From 6abb8c17c610f07efd2661962e028b5763ee0292 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 5 Aug 2021 12:42:02 +0200 Subject: [PATCH 01/32] chore: remove google.cloud.bigquery_v2 code --- google/cloud/bigquery_v2/__init__.py | 46 - google/cloud/bigquery_v2/gapic_metadata.json | 63 - google/cloud/bigquery_v2/py.typed | 2 - google/cloud/bigquery_v2/types/__init__.py | 48 - .../bigquery_v2/types/encryption_config.py | 42 - google/cloud/bigquery_v2/types/model.py | 1507 ----------------- .../bigquery_v2/types/model_reference.py | 44 - .../cloud/bigquery_v2/types/standard_sql.py | 117 -- .../bigquery_v2/types/table_reference.py | 58 - 9 files changed, 1927 deletions(-) delete mode 100644 google/cloud/bigquery_v2/__init__.py delete mode 100644 google/cloud/bigquery_v2/gapic_metadata.json delete mode 100644 google/cloud/bigquery_v2/py.typed delete mode 100644 google/cloud/bigquery_v2/types/__init__.py delete mode 100644 google/cloud/bigquery_v2/types/encryption_config.py delete mode 100644 google/cloud/bigquery_v2/types/model.py delete mode 100644 google/cloud/bigquery_v2/types/model_reference.py delete mode 100644 google/cloud/bigquery_v2/types/standard_sql.py delete mode 100644 google/cloud/bigquery_v2/types/table_reference.py diff --git a/google/cloud/bigquery_v2/__init__.py b/google/cloud/bigquery_v2/__init__.py deleted file mode 100644 index f9957efa9..000000000 --- a/google/cloud/bigquery_v2/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from .types.encryption_config import EncryptionConfiguration -from .types.model import DeleteModelRequest -from .types.model import GetModelRequest -from .types.model import ListModelsRequest -from .types.model import ListModelsResponse -from .types.model import Model -from .types.model import PatchModelRequest -from .types.model_reference import ModelReference -from .types.standard_sql import StandardSqlDataType -from .types.standard_sql import StandardSqlField -from .types.standard_sql import StandardSqlStructType -from .types.standard_sql import StandardSqlTableType -from .types.table_reference import TableReference - -__all__ = ( - "DeleteModelRequest", - "EncryptionConfiguration", - "GetModelRequest", - "ListModelsRequest", - "ListModelsResponse", - "Model", - "ModelReference", - "PatchModelRequest", - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - "TableReference", -) diff --git a/google/cloud/bigquery_v2/gapic_metadata.json b/google/cloud/bigquery_v2/gapic_metadata.json deleted file mode 100644 index 3251a2630..000000000 --- a/google/cloud/bigquery_v2/gapic_metadata.json +++ /dev/null @@ -1,63 +0,0 @@ - { - "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", - "language": "python", - "libraryPackage": "google.cloud.bigquery_v2", - "protoPackage": "google.cloud.bigquery.v2", - "schema": "1.0", - "services": { - "ModelService": { - "clients": { - "grpc": { - "libraryClient": "ModelServiceClient", - "rpcs": { - "DeleteModel": { - "methods": [ - "delete_model" - ] - }, - "GetModel": { - "methods": [ - "get_model" - ] - }, - "ListModels": { - "methods": [ - "list_models" - ] - }, - "PatchModel": { - "methods": [ - "patch_model" - ] - } - } - }, - "grpc-async": { - "libraryClient": "ModelServiceAsyncClient", - "rpcs": { - "DeleteModel": { - "methods": [ - "delete_model" - ] - }, - "GetModel": { - "methods": [ - "get_model" - ] - }, - "ListModels": { - "methods": [ - "list_models" - ] - }, - "PatchModel": { - "methods": [ - "patch_model" - ] - } - } - } - } - } - } -} diff --git a/google/cloud/bigquery_v2/py.typed b/google/cloud/bigquery_v2/py.typed deleted file mode 100644 index e73777993..000000000 --- a/google/cloud/bigquery_v2/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -# Marker file for PEP 561. -# The google-cloud-bigquery package uses inline types. diff --git a/google/cloud/bigquery_v2/types/__init__.py b/google/cloud/bigquery_v2/types/__init__.py deleted file mode 100644 index 83bbb3a54..000000000 --- a/google/cloud/bigquery_v2/types/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from .encryption_config import EncryptionConfiguration -from .model import ( - DeleteModelRequest, - GetModelRequest, - ListModelsRequest, - ListModelsResponse, - Model, - PatchModelRequest, -) -from .model_reference import ModelReference -from .standard_sql import ( - StandardSqlDataType, - StandardSqlField, - StandardSqlStructType, - StandardSqlTableType, -) -from .table_reference import TableReference - -__all__ = ( - "EncryptionConfiguration", - "DeleteModelRequest", - "GetModelRequest", - "ListModelsRequest", - "ListModelsResponse", - "Model", - "PatchModelRequest", - "ModelReference", - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - "TableReference", -) diff --git a/google/cloud/bigquery_v2/types/encryption_config.py b/google/cloud/bigquery_v2/types/encryption_config.py deleted file mode 100644 index 4b9139733..000000000 --- a/google/cloud/bigquery_v2/types/encryption_config.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.protobuf import wrappers_pb2 # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"EncryptionConfiguration",}, -) - - -class EncryptionConfiguration(proto.Message): - r""" - Attributes: - kms_key_name (google.protobuf.wrappers_pb2.StringValue): - Optional. Describes the Cloud KMS encryption - key that will be used to protect destination - BigQuery table. The BigQuery Service Account - associated with your project requires access to - this encryption key. - """ - - kms_key_name = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.StringValue, - ) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py deleted file mode 100644 index 706418401..000000000 --- a/google/cloud/bigquery_v2/types/model.py +++ /dev/null @@ -1,1507 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - -from google.cloud.bigquery_v2.types import encryption_config -from google.cloud.bigquery_v2.types import model_reference as gcb_model_reference -from google.cloud.bigquery_v2.types import standard_sql -from google.cloud.bigquery_v2.types import table_reference -from google.protobuf import timestamp_pb2 # type: ignore -from google.protobuf import wrappers_pb2 # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", - manifest={ - "Model", - "GetModelRequest", - "PatchModelRequest", - "DeleteModelRequest", - "ListModelsRequest", - "ListModelsResponse", - }, -) - - -class Model(proto.Message): - r""" - Attributes: - etag (str): - Output only. A hash of this resource. - model_reference (google.cloud.bigquery_v2.types.ModelReference): - Required. Unique identifier for this model. - creation_time (int): - Output only. The time when this model was - created, in millisecs since the epoch. - last_modified_time (int): - Output only. The time when this model was - last modified, in millisecs since the epoch. - description (str): - Optional. A user-friendly description of this - model. - friendly_name (str): - Optional. A descriptive name for this model. - labels (Sequence[google.cloud.bigquery_v2.types.Model.LabelsEntry]): - The labels associated with this model. You - can use these to organize and group your models. - Label keys and values can be no longer than 63 - characters, can only contain lowercase letters, - numeric characters, underscores and dashes. - International characters are allowed. Label - values are optional. Label keys must start with - a letter and each label in the list must have a - different key. - expiration_time (int): - Optional. The time when this model expires, - in milliseconds since the epoch. If not present, - the model will persist indefinitely. Expired - models will be deleted and their storage - reclaimed. The defaultTableExpirationMs - property of the encapsulating dataset can be - used to set a default expirationTime on newly - created models. - location (str): - Output only. The geographic location where - the model resides. This value is inherited from - the dataset. - encryption_configuration (google.cloud.bigquery_v2.types.EncryptionConfiguration): - Custom encryption configuration (e.g., Cloud - KMS keys). This shows the encryption - configuration of the model data while stored in - BigQuery storage. This field can be used with - PatchModel to update encryption key for an - already encrypted model. - model_type (google.cloud.bigquery_v2.types.Model.ModelType): - Output only. Type of the model resource. - training_runs (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun]): - Output only. Information for all training runs in increasing - order of start_time. - feature_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - Output only. Input feature columns that were - used to train this model. - label_columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - Output only. Label columns that were used to train this - model. The output of the model will have a `predicted_` - prefix to these columns. - best_trial_id (int): - The best trial_id across all training runs. - """ - - class ModelType(proto.Enum): - r"""Indicates the type of the Model.""" - MODEL_TYPE_UNSPECIFIED = 0 - LINEAR_REGRESSION = 1 - LOGISTIC_REGRESSION = 2 - KMEANS = 3 - MATRIX_FACTORIZATION = 4 - DNN_CLASSIFIER = 5 - TENSORFLOW = 6 - DNN_REGRESSOR = 7 - BOOSTED_TREE_REGRESSOR = 9 - BOOSTED_TREE_CLASSIFIER = 10 - ARIMA = 11 - AUTOML_REGRESSOR = 12 - AUTOML_CLASSIFIER = 13 - ARIMA_PLUS = 19 - - class LossType(proto.Enum): - r"""Loss metric to evaluate model training performance.""" - LOSS_TYPE_UNSPECIFIED = 0 - MEAN_SQUARED_LOSS = 1 - MEAN_LOG_LOSS = 2 - - class DistanceType(proto.Enum): - r"""Distance metric used to compute the distance between two - points. - """ - DISTANCE_TYPE_UNSPECIFIED = 0 - EUCLIDEAN = 1 - COSINE = 2 - - class DataSplitMethod(proto.Enum): - r"""Indicates the method to split input data into multiple - tables. - """ - DATA_SPLIT_METHOD_UNSPECIFIED = 0 - RANDOM = 1 - CUSTOM = 2 - SEQUENTIAL = 3 - NO_SPLIT = 4 - AUTO_SPLIT = 5 - - class DataFrequency(proto.Enum): - r"""Type of supported data frequency for time series forecasting - models. - """ - DATA_FREQUENCY_UNSPECIFIED = 0 - AUTO_FREQUENCY = 1 - YEARLY = 2 - QUARTERLY = 3 - MONTHLY = 4 - WEEKLY = 5 - DAILY = 6 - HOURLY = 7 - PER_MINUTE = 8 - - class HolidayRegion(proto.Enum): - r"""Type of supported holiday regions for time series forecasting - models. - """ - HOLIDAY_REGION_UNSPECIFIED = 0 - GLOBAL = 1 - NA = 2 - JAPAC = 3 - EMEA = 4 - LAC = 5 - AE = 6 - AR = 7 - AT = 8 - AU = 9 - BE = 10 - BR = 11 - CA = 12 - CH = 13 - CL = 14 - CN = 15 - CO = 16 - CS = 17 - CZ = 18 - DE = 19 - DK = 20 - DZ = 21 - EC = 22 - EE = 23 - EG = 24 - ES = 25 - FI = 26 - FR = 27 - GB = 28 - GR = 29 - HK = 30 - HU = 31 - ID = 32 - IE = 33 - IL = 34 - IN = 35 - IR = 36 - IT = 37 - JP = 38 - KR = 39 - LV = 40 - MA = 41 - MX = 42 - MY = 43 - NG = 44 - NL = 45 - NO = 46 - NZ = 47 - PE = 48 - PH = 49 - PK = 50 - PL = 51 - PT = 52 - RO = 53 - RS = 54 - RU = 55 - SA = 56 - SE = 57 - SG = 58 - SI = 59 - SK = 60 - TH = 61 - TR = 62 - TW = 63 - UA = 64 - US = 65 - VE = 66 - VN = 67 - ZA = 68 - - class LearnRateStrategy(proto.Enum): - r"""Indicates the learning rate optimization strategy to use.""" - LEARN_RATE_STRATEGY_UNSPECIFIED = 0 - LINE_SEARCH = 1 - CONSTANT = 2 - - class OptimizationStrategy(proto.Enum): - r"""Indicates the optimization strategy used for training.""" - OPTIMIZATION_STRATEGY_UNSPECIFIED = 0 - BATCH_GRADIENT_DESCENT = 1 - NORMAL_EQUATION = 2 - - class FeedbackType(proto.Enum): - r"""Indicates the training algorithm to use for matrix - factorization models. - """ - FEEDBACK_TYPE_UNSPECIFIED = 0 - IMPLICIT = 1 - EXPLICIT = 2 - - class SeasonalPeriod(proto.Message): - r""" """ - - class SeasonalPeriodType(proto.Enum): - r"""""" - SEASONAL_PERIOD_TYPE_UNSPECIFIED = 0 - NO_SEASONALITY = 1 - DAILY = 2 - WEEKLY = 3 - MONTHLY = 4 - QUARTERLY = 5 - YEARLY = 6 - - class KmeansEnums(proto.Message): - r""" """ - - class KmeansInitializationMethod(proto.Enum): - r"""Indicates the method used to initialize the centroids for - KMeans clustering algorithm. - """ - KMEANS_INITIALIZATION_METHOD_UNSPECIFIED = 0 - RANDOM = 1 - CUSTOM = 2 - KMEANS_PLUS_PLUS = 3 - - class RegressionMetrics(proto.Message): - r"""Evaluation metrics for regression and explicit feedback type - matrix factorization models. - - Attributes: - mean_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean absolute error. - mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean squared error. - mean_squared_log_error (google.protobuf.wrappers_pb2.DoubleValue): - Mean squared log error. - median_absolute_error (google.protobuf.wrappers_pb2.DoubleValue): - Median absolute error. - r_squared (google.protobuf.wrappers_pb2.DoubleValue): - R^2 score. This corresponds to r2_score in ML.EVALUATE. - """ - - mean_absolute_error = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_error = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - mean_squared_log_error = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - median_absolute_error = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - r_squared = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - - class AggregateClassificationMetrics(proto.Message): - r"""Aggregate metrics for classification/classifier models. For - multi-class models, the metrics are either macro-averaged or - micro-averaged. When macro-averaged, the metrics are calculated - for each label and then an unweighted average is taken of those - values. When micro-averaged, the metric is calculated globally - by counting the total number of correctly predicted rows. - - Attributes: - precision (google.protobuf.wrappers_pb2.DoubleValue): - Precision is the fraction of actual positive - predictions that had positive actual labels. For - multiclass this is a macro-averaged metric - treating each class as a binary classifier. - recall (google.protobuf.wrappers_pb2.DoubleValue): - Recall is the fraction of actual positive - labels that were given a positive prediction. - For multiclass this is a macro-averaged metric. - accuracy (google.protobuf.wrappers_pb2.DoubleValue): - Accuracy is the fraction of predictions given - the correct label. For multiclass this is a - micro-averaged metric. - threshold (google.protobuf.wrappers_pb2.DoubleValue): - Threshold at which the metrics are computed. - For binary classification models this is the - positive class threshold. For multi-class - classfication models this is the confidence - threshold. - f1_score (google.protobuf.wrappers_pb2.DoubleValue): - The F1 score is an average of recall and - precision. For multiclass this is a macro- - averaged metric. - log_loss (google.protobuf.wrappers_pb2.DoubleValue): - Logarithmic Loss. For multiclass this is a - macro-averaged metric. - roc_auc (google.protobuf.wrappers_pb2.DoubleValue): - Area Under a ROC Curve. For multiclass this - is a macro-averaged metric. - """ - - precision = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - recall = proto.Field(proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue,) - accuracy = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - threshold = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - f1_score = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - log_loss = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - roc_auc = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.DoubleValue, - ) - - class BinaryClassificationMetrics(proto.Message): - r"""Evaluation metrics for binary classification/classifier - models. - - Attributes: - aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): - Aggregate classification metrics. - binary_confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics.BinaryConfusionMatrix]): - Binary confusion matrix at multiple - thresholds. - positive_label (str): - Label representing the positive class. - negative_label (str): - Label representing the negative class. - """ - - class BinaryConfusionMatrix(proto.Message): - r"""Confusion matrix for binary classification models. - Attributes: - positive_class_threshold (google.protobuf.wrappers_pb2.DoubleValue): - Threshold value used when computing each of - the following metric. - true_positives (google.protobuf.wrappers_pb2.Int64Value): - Number of true samples predicted as true. - false_positives (google.protobuf.wrappers_pb2.Int64Value): - Number of false samples predicted as true. - true_negatives (google.protobuf.wrappers_pb2.Int64Value): - Number of true samples predicted as false. - false_negatives (google.protobuf.wrappers_pb2.Int64Value): - Number of false samples predicted as false. - precision (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of actual positive predictions - that had positive actual labels. - recall (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of actual positive labels that - were given a positive prediction. - f1_score (google.protobuf.wrappers_pb2.DoubleValue): - The equally weighted average of recall and - precision. - accuracy (google.protobuf.wrappers_pb2.DoubleValue): - The fraction of predictions given the correct - label. - """ - - positive_class_threshold = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - true_positives = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - false_positives = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - true_negatives = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.Int64Value, - ) - false_negatives = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.Int64Value, - ) - precision = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - recall = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.DoubleValue, - ) - f1_score = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.DoubleValue, - ) - accuracy = proto.Field( - proto.MESSAGE, number=9, message=wrappers_pb2.DoubleValue, - ) - - aggregate_classification_metrics = proto.Field( - proto.MESSAGE, number=1, message="Model.AggregateClassificationMetrics", - ) - binary_confusion_matrix_list = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.BinaryClassificationMetrics.BinaryConfusionMatrix", - ) - positive_label = proto.Field(proto.STRING, number=3,) - negative_label = proto.Field(proto.STRING, number=4,) - - class MultiClassClassificationMetrics(proto.Message): - r"""Evaluation metrics for multi-class classification/classifier - models. - - Attributes: - aggregate_classification_metrics (google.cloud.bigquery_v2.types.Model.AggregateClassificationMetrics): - Aggregate classification metrics. - confusion_matrix_list (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix]): - Confusion matrix at different thresholds. - """ - - class ConfusionMatrix(proto.Message): - r"""Confusion matrix for multi-class classification models. - Attributes: - confidence_threshold (google.protobuf.wrappers_pb2.DoubleValue): - Confidence threshold used when computing the - entries of the confusion matrix. - rows (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Row]): - One row per actual label. - """ - - class Entry(proto.Message): - r"""A single entry in the confusion matrix. - Attributes: - predicted_label (str): - The predicted label. For confidence_threshold > 0, we will - also add an entry indicating the number of items under the - confidence threshold. - item_count (google.protobuf.wrappers_pb2.Int64Value): - Number of items being predicted as this - label. - """ - - predicted_label = proto.Field(proto.STRING, number=1,) - item_count = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - - class Row(proto.Message): - r"""A single row in the confusion matrix. - Attributes: - actual_label (str): - The original label of this row. - entries (Sequence[google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry]): - Info describing predicted label distribution. - """ - - actual_label = proto.Field(proto.STRING, number=1,) - entries = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix.Entry", - ) - - confidence_threshold = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - rows = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix.Row", - ) - - aggregate_classification_metrics = proto.Field( - proto.MESSAGE, number=1, message="Model.AggregateClassificationMetrics", - ) - confusion_matrix_list = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.MultiClassClassificationMetrics.ConfusionMatrix", - ) - - class ClusteringMetrics(proto.Message): - r"""Evaluation metrics for clustering models. - Attributes: - davies_bouldin_index (google.protobuf.wrappers_pb2.DoubleValue): - Davies-Bouldin index. - mean_squared_distance (google.protobuf.wrappers_pb2.DoubleValue): - Mean of squared distances between each sample - to its cluster centroid. - clusters (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster]): - Information for all clusters. - """ - - class Cluster(proto.Message): - r"""Message containing the information about one cluster. - Attributes: - centroid_id (int): - Centroid id. - feature_values (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue]): - Values of highly variant features for this - cluster. - count (google.protobuf.wrappers_pb2.Int64Value): - Count of training data rows that were - assigned to this cluster. - """ - - class FeatureValue(proto.Message): - r"""Representative value of a single feature within the cluster. - Attributes: - feature_column (str): - The feature column name. - numerical_value (google.protobuf.wrappers_pb2.DoubleValue): - The numerical feature value. This is the - centroid value for this feature. - categorical_value (google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): - The categorical feature value. - """ - - class CategoricalValue(proto.Message): - r"""Representative value of a categorical feature. - Attributes: - category_counts (Sequence[google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount]): - Counts of all categories for the categorical feature. If - there are more than ten categories, we return top ten (by - count) and return one more CategoryCount with category - "*OTHER*" and count as aggregate counts of remaining - categories. - """ - - class CategoryCount(proto.Message): - r"""Represents the count of a single category within the cluster. - Attributes: - category (str): - The name of category. - count (google.protobuf.wrappers_pb2.Int64Value): - The count of training samples matching the - category within the cluster. - """ - - category = proto.Field(proto.STRING, number=1,) - count = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value, - ) - - category_counts = proto.RepeatedField( - proto.MESSAGE, - number=1, - message="Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue.CategoryCount", - ) - - feature_column = proto.Field(proto.STRING, number=1,) - numerical_value = proto.Field( - proto.MESSAGE, - number=2, - oneof="value", - message=wrappers_pb2.DoubleValue, - ) - categorical_value = proto.Field( - proto.MESSAGE, - number=3, - oneof="value", - message="Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue", - ) - - centroid_id = proto.Field(proto.INT64, number=1,) - feature_values = proto.RepeatedField( - proto.MESSAGE, - number=2, - message="Model.ClusteringMetrics.Cluster.FeatureValue", - ) - count = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - - davies_bouldin_index = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_distance = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - clusters = proto.RepeatedField( - proto.MESSAGE, number=3, message="Model.ClusteringMetrics.Cluster", - ) - - class RankingMetrics(proto.Message): - r"""Evaluation metrics used by weighted-ALS models specified by - feedback_type=implicit. - - Attributes: - mean_average_precision (google.protobuf.wrappers_pb2.DoubleValue): - Calculates a precision per user for all the - items by ranking them and then averages all the - precisions across all the users. - mean_squared_error (google.protobuf.wrappers_pb2.DoubleValue): - Similar to the mean squared error computed in - regression and explicit recommendation models - except instead of computing the rating directly, - the output from evaluate is computed against a - preference which is 1 or 0 depending on if the - rating exists or not. - normalized_discounted_cumulative_gain (google.protobuf.wrappers_pb2.DoubleValue): - A metric to determine the goodness of a - ranking calculated from the predicted confidence - by comparing it to an ideal rank measured by the - original ratings. - average_rank (google.protobuf.wrappers_pb2.DoubleValue): - Determines the goodness of a ranking by - computing the percentile rank from the predicted - confidence and dividing it by the original rank. - """ - - mean_average_precision = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.DoubleValue, - ) - mean_squared_error = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - normalized_discounted_cumulative_gain = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.DoubleValue, - ) - average_rank = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - - class ArimaForecastingMetrics(proto.Message): - r"""Model evaluation metrics for ARIMA forecasting models. - Attributes: - non_seasonal_order (Sequence[google.cloud.bigquery_v2.types.Model.ArimaOrder]): - Non-seasonal order. - arima_fitting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics]): - Arima model fitting metrics. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_drift (Sequence[bool]): - Whether Arima model fitted with drift or not. - It is always false when d is not 1. - time_series_id (Sequence[str]): - Id to differentiate different time series for - the large-scale case. - arima_single_model_forecasting_metrics (Sequence[google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics]): - Repeated as there can be many metric sets - (one for each model) in auto-arima and the - large-scale case. - """ - - class ArimaSingleModelForecastingMetrics(proto.Message): - r"""Model evaluation metrics for a single ARIMA forecasting - model. - - Attributes: - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - Non-seasonal order. - arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): - Arima fitting metrics. - has_drift (bool): - Is arima model fitted with drift or not. It - is always false when d is not 1. - time_series_id (str): - The time_series_id value for this time series. It will be - one of the unique values from the time_series_id_column - specified during ARIMA model training. Only present when - time_series_id_column training option was used. - time_series_ids (Sequence[str]): - The tuple of time_series_ids identifying this time series. - It will be one of the unique tuples of values present in the - time_series_id_columns specified during ARIMA model - training. Only present when time_series_id_columns training - option was used and the order of values here are same as the - order of time_series_id_columns. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_holiday_effect (google.protobuf.wrappers_pb2.BoolValue): - If true, holiday_effect is a part of time series - decomposition result. - has_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, spikes_and_dips is a part of time series - decomposition result. - has_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, step_changes is a part of time series decomposition - result. - """ - - non_seasonal_order = proto.Field( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_fitting_metrics = proto.Field( - proto.MESSAGE, number=2, message="Model.ArimaFittingMetrics", - ) - has_drift = proto.Field(proto.BOOL, number=3,) - time_series_id = proto.Field(proto.STRING, number=4,) - time_series_ids = proto.RepeatedField(proto.STRING, number=9,) - seasonal_periods = proto.RepeatedField( - proto.ENUM, number=5, enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_holiday_effect = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.BoolValue, - ) - has_spikes_and_dips = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - has_step_changes = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - - non_seasonal_order = proto.RepeatedField( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_fitting_metrics = proto.RepeatedField( - proto.MESSAGE, number=2, message="Model.ArimaFittingMetrics", - ) - seasonal_periods = proto.RepeatedField( - proto.ENUM, number=3, enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_drift = proto.RepeatedField(proto.BOOL, number=4,) - time_series_id = proto.RepeatedField(proto.STRING, number=5,) - arima_single_model_forecasting_metrics = proto.RepeatedField( - proto.MESSAGE, - number=6, - message="Model.ArimaForecastingMetrics.ArimaSingleModelForecastingMetrics", - ) - - class EvaluationMetrics(proto.Message): - r"""Evaluation metrics of a model. These are either computed on - all training data or just the eval data based on whether eval - data was used during training. These are not present for - imported models. - - Attributes: - regression_metrics (google.cloud.bigquery_v2.types.Model.RegressionMetrics): - Populated for regression models and explicit - feedback type matrix factorization models. - binary_classification_metrics (google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics): - Populated for binary - classification/classifier models. - multi_class_classification_metrics (google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics): - Populated for multi-class - classification/classifier models. - clustering_metrics (google.cloud.bigquery_v2.types.Model.ClusteringMetrics): - Populated for clustering models. - ranking_metrics (google.cloud.bigquery_v2.types.Model.RankingMetrics): - Populated for implicit feedback type matrix - factorization models. - arima_forecasting_metrics (google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics): - Populated for ARIMA models. - """ - - regression_metrics = proto.Field( - proto.MESSAGE, number=1, oneof="metrics", message="Model.RegressionMetrics", - ) - binary_classification_metrics = proto.Field( - proto.MESSAGE, - number=2, - oneof="metrics", - message="Model.BinaryClassificationMetrics", - ) - multi_class_classification_metrics = proto.Field( - proto.MESSAGE, - number=3, - oneof="metrics", - message="Model.MultiClassClassificationMetrics", - ) - clustering_metrics = proto.Field( - proto.MESSAGE, number=4, oneof="metrics", message="Model.ClusteringMetrics", - ) - ranking_metrics = proto.Field( - proto.MESSAGE, number=5, oneof="metrics", message="Model.RankingMetrics", - ) - arima_forecasting_metrics = proto.Field( - proto.MESSAGE, - number=6, - oneof="metrics", - message="Model.ArimaForecastingMetrics", - ) - - class DataSplitResult(proto.Message): - r"""Data split result. This contains references to the training - and evaluation data tables that were used to train the model. - - Attributes: - training_table (google.cloud.bigquery_v2.types.TableReference): - Table reference of the training data after - split. - evaluation_table (google.cloud.bigquery_v2.types.TableReference): - Table reference of the evaluation data after - split. - """ - - training_table = proto.Field( - proto.MESSAGE, number=1, message=table_reference.TableReference, - ) - evaluation_table = proto.Field( - proto.MESSAGE, number=2, message=table_reference.TableReference, - ) - - class ArimaOrder(proto.Message): - r"""Arima order, can be used for both non-seasonal and seasonal - parts. - - Attributes: - p (int): - Order of the autoregressive part. - d (int): - Order of the differencing part. - q (int): - Order of the moving-average part. - """ - - p = proto.Field(proto.INT64, number=1,) - d = proto.Field(proto.INT64, number=2,) - q = proto.Field(proto.INT64, number=3,) - - class ArimaFittingMetrics(proto.Message): - r"""ARIMA model fitting metrics. - Attributes: - log_likelihood (float): - Log-likelihood. - aic (float): - AIC. - variance (float): - Variance. - """ - - log_likelihood = proto.Field(proto.DOUBLE, number=1,) - aic = proto.Field(proto.DOUBLE, number=2,) - variance = proto.Field(proto.DOUBLE, number=3,) - - class GlobalExplanation(proto.Message): - r"""Global explanations containing the top most important - features after training. - - Attributes: - explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation.Explanation]): - A list of the top global explanations. Sorted - by absolute value of attribution in descending - order. - class_label (str): - Class label for this set of global - explanations. Will be empty/null for binary - logistic and linear regression models. Sorted - alphabetically in descending order. - """ - - class Explanation(proto.Message): - r"""Explanation for a single feature. - Attributes: - feature_name (str): - Full name of the feature. For non-numerical features, will - be formatted like .. - Overall size of feature name will always be truncated to - first 120 characters. - attribution (google.protobuf.wrappers_pb2.DoubleValue): - Attribution of feature. - """ - - feature_name = proto.Field(proto.STRING, number=1,) - attribution = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - - explanations = proto.RepeatedField( - proto.MESSAGE, number=1, message="Model.GlobalExplanation.Explanation", - ) - class_label = proto.Field(proto.STRING, number=2,) - - class TrainingRun(proto.Message): - r"""Information about a single training query run for the model. - Attributes: - training_options (google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions): - Options that were used for this training run, - includes user specified and default options that - were used. - start_time (google.protobuf.timestamp_pb2.Timestamp): - The start time of this training run. - results (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult]): - Output of each iteration run, results.size() <= - max_iterations. - evaluation_metrics (google.cloud.bigquery_v2.types.Model.EvaluationMetrics): - The evaluation metrics over training/eval - data that were computed at the end of training. - data_split_result (google.cloud.bigquery_v2.types.Model.DataSplitResult): - Data split result of the training run. Only - set when the input data is actually split. - global_explanations (Sequence[google.cloud.bigquery_v2.types.Model.GlobalExplanation]): - Global explanations for important features of - the model. For multi-class models, there is one - entry for each label class. For other models, - there is only one entry in the list. - """ - - class TrainingOptions(proto.Message): - r"""Options used in model training. - Attributes: - max_iterations (int): - The maximum number of iterations in training. - Used only for iterative training algorithms. - loss_type (google.cloud.bigquery_v2.types.Model.LossType): - Type of loss function used during training - run. - learn_rate (float): - Learning rate in training. Used only for - iterative training algorithms. - l1_regularization (google.protobuf.wrappers_pb2.DoubleValue): - L1 regularization coefficient. - l2_regularization (google.protobuf.wrappers_pb2.DoubleValue): - L2 regularization coefficient. - min_relative_progress (google.protobuf.wrappers_pb2.DoubleValue): - When early_stop is true, stops training when accuracy - improvement is less than 'min_relative_progress'. Used only - for iterative training algorithms. - warm_start (google.protobuf.wrappers_pb2.BoolValue): - Whether to train a model from the last - checkpoint. - early_stop (google.protobuf.wrappers_pb2.BoolValue): - Whether to stop early when the loss doesn't improve - significantly any more (compared to min_relative_progress). - Used only for iterative training algorithms. - input_label_columns (Sequence[str]): - Name of input label columns in training data. - data_split_method (google.cloud.bigquery_v2.types.Model.DataSplitMethod): - The data split type for training and - evaluation, e.g. RANDOM. - data_split_eval_fraction (float): - The fraction of evaluation data over the - whole input data. The rest of data will be used - as training data. The format should be double. - Accurate to two decimal places. - Default value is 0.2. - data_split_column (str): - The column to split data with. This column won't be used as - a feature. - - 1. When data_split_method is CUSTOM, the corresponding - column should be boolean. The rows with true value tag - are eval data, and the false are training data. - 2. When data_split_method is SEQ, the first - DATA_SPLIT_EVAL_FRACTION rows (from smallest to largest) - in the corresponding column are used as training data, - and the rest are eval data. It respects the order in - Orderable data types: - https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#data-type-properties - learn_rate_strategy (google.cloud.bigquery_v2.types.Model.LearnRateStrategy): - The strategy to determine learn rate for the - current iteration. - initial_learn_rate (float): - Specifies the initial learning rate for the - line search learn rate strategy. - label_class_weights (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.TrainingOptions.LabelClassWeightsEntry]): - Weights associated with each label class, for - rebalancing the training data. Only applicable - for classification models. - user_column (str): - User column specified for matrix - factorization models. - item_column (str): - Item column specified for matrix - factorization models. - distance_type (google.cloud.bigquery_v2.types.Model.DistanceType): - Distance type for clustering models. - num_clusters (int): - Number of clusters for clustering models. - model_uri (str): - Google Cloud Storage URI from which the model - was imported. Only applicable for imported - models. - optimization_strategy (google.cloud.bigquery_v2.types.Model.OptimizationStrategy): - Optimization strategy for training linear - regression models. - hidden_units (Sequence[int]): - Hidden units for dnn models. - batch_size (int): - Batch size for dnn models. - dropout (google.protobuf.wrappers_pb2.DoubleValue): - Dropout probability for dnn models. - max_tree_depth (int): - Maximum depth of a tree for boosted tree - models. - subsample (float): - Subsample fraction of the training data to - grow tree to prevent overfitting for boosted - tree models. - min_split_loss (google.protobuf.wrappers_pb2.DoubleValue): - Minimum split loss for boosted tree models. - num_factors (int): - Num factors specified for matrix - factorization models. - feedback_type (google.cloud.bigquery_v2.types.Model.FeedbackType): - Feedback type that specifies which algorithm - to run for matrix factorization. - wals_alpha (google.protobuf.wrappers_pb2.DoubleValue): - Hyperparameter for matrix factoration when - implicit feedback type is specified. - kmeans_initialization_method (google.cloud.bigquery_v2.types.Model.KmeansEnums.KmeansInitializationMethod): - The method used to initialize the centroids - for kmeans algorithm. - kmeans_initialization_column (str): - The column used to provide the initial centroids for kmeans - algorithm when kmeans_initialization_method is CUSTOM. - time_series_timestamp_column (str): - Column to be designated as time series - timestamp for ARIMA model. - time_series_data_column (str): - Column to be designated as time series data - for ARIMA model. - auto_arima (bool): - Whether to enable auto ARIMA or not. - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - A specification of the non-seasonal part of - the ARIMA model: the three components (p, d, q) - are the AR order, the degree of differencing, - and the MA order. - data_frequency (google.cloud.bigquery_v2.types.Model.DataFrequency): - The data frequency of a time series. - include_drift (bool): - Include drift when fitting an ARIMA model. - holiday_region (google.cloud.bigquery_v2.types.Model.HolidayRegion): - The geographical region based on which the - holidays are considered in time series modeling. - If a valid value is specified, then holiday - effects modeling is enabled. - time_series_id_column (str): - The time series id column that was used - during ARIMA model training. - time_series_id_columns (Sequence[str]): - The time series id columns that were used - during ARIMA model training. - horizon (int): - The number of periods ahead that need to be - forecasted. - preserve_input_structs (bool): - Whether to preserve the input structs in output feature - names. Suppose there is a struct A with field b. When false - (default), the output feature name is A_b. When true, the - output feature name is A.b. - auto_arima_max_order (int): - The max value of non-seasonal p and q. - decompose_time_series (google.protobuf.wrappers_pb2.BoolValue): - If true, perform decompose time series and - save the results. - clean_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, clean spikes and dips in the input - time series. - adjust_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, detect step changes and make data - adjustment in the input time series. - """ - - max_iterations = proto.Field(proto.INT64, number=1,) - loss_type = proto.Field(proto.ENUM, number=2, enum="Model.LossType",) - learn_rate = proto.Field(proto.DOUBLE, number=3,) - l1_regularization = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.DoubleValue, - ) - l2_regularization = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - min_relative_progress = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - warm_start = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - early_stop = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - input_label_columns = proto.RepeatedField(proto.STRING, number=9,) - data_split_method = proto.Field( - proto.ENUM, number=10, enum="Model.DataSplitMethod", - ) - data_split_eval_fraction = proto.Field(proto.DOUBLE, number=11,) - data_split_column = proto.Field(proto.STRING, number=12,) - learn_rate_strategy = proto.Field( - proto.ENUM, number=13, enum="Model.LearnRateStrategy", - ) - initial_learn_rate = proto.Field(proto.DOUBLE, number=16,) - label_class_weights = proto.MapField(proto.STRING, proto.DOUBLE, number=17,) - user_column = proto.Field(proto.STRING, number=18,) - item_column = proto.Field(proto.STRING, number=19,) - distance_type = proto.Field( - proto.ENUM, number=20, enum="Model.DistanceType", - ) - num_clusters = proto.Field(proto.INT64, number=21,) - model_uri = proto.Field(proto.STRING, number=22,) - optimization_strategy = proto.Field( - proto.ENUM, number=23, enum="Model.OptimizationStrategy", - ) - hidden_units = proto.RepeatedField(proto.INT64, number=24,) - batch_size = proto.Field(proto.INT64, number=25,) - dropout = proto.Field( - proto.MESSAGE, number=26, message=wrappers_pb2.DoubleValue, - ) - max_tree_depth = proto.Field(proto.INT64, number=27,) - subsample = proto.Field(proto.DOUBLE, number=28,) - min_split_loss = proto.Field( - proto.MESSAGE, number=29, message=wrappers_pb2.DoubleValue, - ) - num_factors = proto.Field(proto.INT64, number=30,) - feedback_type = proto.Field( - proto.ENUM, number=31, enum="Model.FeedbackType", - ) - wals_alpha = proto.Field( - proto.MESSAGE, number=32, message=wrappers_pb2.DoubleValue, - ) - kmeans_initialization_method = proto.Field( - proto.ENUM, - number=33, - enum="Model.KmeansEnums.KmeansInitializationMethod", - ) - kmeans_initialization_column = proto.Field(proto.STRING, number=34,) - time_series_timestamp_column = proto.Field(proto.STRING, number=35,) - time_series_data_column = proto.Field(proto.STRING, number=36,) - auto_arima = proto.Field(proto.BOOL, number=37,) - non_seasonal_order = proto.Field( - proto.MESSAGE, number=38, message="Model.ArimaOrder", - ) - data_frequency = proto.Field( - proto.ENUM, number=39, enum="Model.DataFrequency", - ) - include_drift = proto.Field(proto.BOOL, number=41,) - holiday_region = proto.Field( - proto.ENUM, number=42, enum="Model.HolidayRegion", - ) - time_series_id_column = proto.Field(proto.STRING, number=43,) - time_series_id_columns = proto.RepeatedField(proto.STRING, number=51,) - horizon = proto.Field(proto.INT64, number=44,) - preserve_input_structs = proto.Field(proto.BOOL, number=45,) - auto_arima_max_order = proto.Field(proto.INT64, number=46,) - decompose_time_series = proto.Field( - proto.MESSAGE, number=50, message=wrappers_pb2.BoolValue, - ) - clean_spikes_and_dips = proto.Field( - proto.MESSAGE, number=52, message=wrappers_pb2.BoolValue, - ) - adjust_step_changes = proto.Field( - proto.MESSAGE, number=53, message=wrappers_pb2.BoolValue, - ) - - class IterationResult(proto.Message): - r"""Information about a single iteration of the training run. - Attributes: - index (google.protobuf.wrappers_pb2.Int32Value): - Index of the iteration, 0 based. - duration_ms (google.protobuf.wrappers_pb2.Int64Value): - Time taken to run the iteration in - milliseconds. - training_loss (google.protobuf.wrappers_pb2.DoubleValue): - Loss computed on the training data at the end - of iteration. - eval_loss (google.protobuf.wrappers_pb2.DoubleValue): - Loss computed on the eval data at the end of - iteration. - learn_rate (float): - Learn rate used for this iteration. - cluster_infos (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ClusterInfo]): - Information about top clusters for clustering - models. - arima_result (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult): - - """ - - class ClusterInfo(proto.Message): - r"""Information about a single cluster for clustering model. - Attributes: - centroid_id (int): - Centroid id. - cluster_radius (google.protobuf.wrappers_pb2.DoubleValue): - Cluster radius, the average distance from - centroid to each point assigned to the cluster. - cluster_size (google.protobuf.wrappers_pb2.Int64Value): - Cluster size, the total number of points - assigned to the cluster. - """ - - centroid_id = proto.Field(proto.INT64, number=1,) - cluster_radius = proto.Field( - proto.MESSAGE, number=2, message=wrappers_pb2.DoubleValue, - ) - cluster_size = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.Int64Value, - ) - - class ArimaResult(proto.Message): - r"""(Auto-)arima fitting result. Wrap everything in ArimaResult - for easier refactoring if we want to use model-specific - iteration results. - - Attributes: - arima_model_info (Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo]): - This message is repeated because there are - multiple arima models fitted in auto-arima. For - non-auto-arima model, its size is one. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - """ - - class ArimaCoefficients(proto.Message): - r"""Arima coefficients. - Attributes: - auto_regressive_coefficients (Sequence[float]): - Auto-regressive coefficients, an array of - double. - moving_average_coefficients (Sequence[float]): - Moving-average coefficients, an array of - double. - intercept_coefficient (float): - Intercept coefficient, just a double not an - array. - """ - - auto_regressive_coefficients = proto.RepeatedField( - proto.DOUBLE, number=1, - ) - moving_average_coefficients = proto.RepeatedField( - proto.DOUBLE, number=2, - ) - intercept_coefficient = proto.Field(proto.DOUBLE, number=3,) - - class ArimaModelInfo(proto.Message): - r"""Arima model information. - Attributes: - non_seasonal_order (google.cloud.bigquery_v2.types.Model.ArimaOrder): - Non-seasonal order. - arima_coefficients (google.cloud.bigquery_v2.types.Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients): - Arima coefficients. - arima_fitting_metrics (google.cloud.bigquery_v2.types.Model.ArimaFittingMetrics): - Arima fitting metrics. - has_drift (bool): - Whether Arima model fitted with drift or not. - It is always false when d is not 1. - time_series_id (str): - The time_series_id value for this time series. It will be - one of the unique values from the time_series_id_column - specified during ARIMA model training. Only present when - time_series_id_column training option was used. - time_series_ids (Sequence[str]): - The tuple of time_series_ids identifying this time series. - It will be one of the unique tuples of values present in the - time_series_id_columns specified during ARIMA model - training. Only present when time_series_id_columns training - option was used and the order of values here are same as the - order of time_series_id_columns. - seasonal_periods (Sequence[google.cloud.bigquery_v2.types.Model.SeasonalPeriod.SeasonalPeriodType]): - Seasonal periods. Repeated because multiple - periods are supported for one time series. - has_holiday_effect (google.protobuf.wrappers_pb2.BoolValue): - If true, holiday_effect is a part of time series - decomposition result. - has_spikes_and_dips (google.protobuf.wrappers_pb2.BoolValue): - If true, spikes_and_dips is a part of time series - decomposition result. - has_step_changes (google.protobuf.wrappers_pb2.BoolValue): - If true, step_changes is a part of time series decomposition - result. - """ - - non_seasonal_order = proto.Field( - proto.MESSAGE, number=1, message="Model.ArimaOrder", - ) - arima_coefficients = proto.Field( - proto.MESSAGE, - number=2, - message="Model.TrainingRun.IterationResult.ArimaResult.ArimaCoefficients", - ) - arima_fitting_metrics = proto.Field( - proto.MESSAGE, number=3, message="Model.ArimaFittingMetrics", - ) - has_drift = proto.Field(proto.BOOL, number=4,) - time_series_id = proto.Field(proto.STRING, number=5,) - time_series_ids = proto.RepeatedField(proto.STRING, number=10,) - seasonal_periods = proto.RepeatedField( - proto.ENUM, - number=6, - enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - has_holiday_effect = proto.Field( - proto.MESSAGE, number=7, message=wrappers_pb2.BoolValue, - ) - has_spikes_and_dips = proto.Field( - proto.MESSAGE, number=8, message=wrappers_pb2.BoolValue, - ) - has_step_changes = proto.Field( - proto.MESSAGE, number=9, message=wrappers_pb2.BoolValue, - ) - - arima_model_info = proto.RepeatedField( - proto.MESSAGE, - number=1, - message="Model.TrainingRun.IterationResult.ArimaResult.ArimaModelInfo", - ) - seasonal_periods = proto.RepeatedField( - proto.ENUM, - number=2, - enum="Model.SeasonalPeriod.SeasonalPeriodType", - ) - - index = proto.Field( - proto.MESSAGE, number=1, message=wrappers_pb2.Int32Value, - ) - duration_ms = proto.Field( - proto.MESSAGE, number=4, message=wrappers_pb2.Int64Value, - ) - training_loss = proto.Field( - proto.MESSAGE, number=5, message=wrappers_pb2.DoubleValue, - ) - eval_loss = proto.Field( - proto.MESSAGE, number=6, message=wrappers_pb2.DoubleValue, - ) - learn_rate = proto.Field(proto.DOUBLE, number=7,) - cluster_infos = proto.RepeatedField( - proto.MESSAGE, - number=8, - message="Model.TrainingRun.IterationResult.ClusterInfo", - ) - arima_result = proto.Field( - proto.MESSAGE, - number=9, - message="Model.TrainingRun.IterationResult.ArimaResult", - ) - - training_options = proto.Field( - proto.MESSAGE, number=1, message="Model.TrainingRun.TrainingOptions", - ) - start_time = proto.Field( - proto.MESSAGE, number=8, message=timestamp_pb2.Timestamp, - ) - results = proto.RepeatedField( - proto.MESSAGE, number=6, message="Model.TrainingRun.IterationResult", - ) - evaluation_metrics = proto.Field( - proto.MESSAGE, number=7, message="Model.EvaluationMetrics", - ) - data_split_result = proto.Field( - proto.MESSAGE, number=9, message="Model.DataSplitResult", - ) - global_explanations = proto.RepeatedField( - proto.MESSAGE, number=10, message="Model.GlobalExplanation", - ) - - etag = proto.Field(proto.STRING, number=1,) - model_reference = proto.Field( - proto.MESSAGE, number=2, message=gcb_model_reference.ModelReference, - ) - creation_time = proto.Field(proto.INT64, number=5,) - last_modified_time = proto.Field(proto.INT64, number=6,) - description = proto.Field(proto.STRING, number=12,) - friendly_name = proto.Field(proto.STRING, number=14,) - labels = proto.MapField(proto.STRING, proto.STRING, number=15,) - expiration_time = proto.Field(proto.INT64, number=16,) - location = proto.Field(proto.STRING, number=13,) - encryption_configuration = proto.Field( - proto.MESSAGE, number=17, message=encryption_config.EncryptionConfiguration, - ) - model_type = proto.Field(proto.ENUM, number=7, enum=ModelType,) - training_runs = proto.RepeatedField(proto.MESSAGE, number=9, message=TrainingRun,) - feature_columns = proto.RepeatedField( - proto.MESSAGE, number=10, message=standard_sql.StandardSqlField, - ) - label_columns = proto.RepeatedField( - proto.MESSAGE, number=11, message=standard_sql.StandardSqlField, - ) - best_trial_id = proto.Field(proto.INT64, number=19,) - - -class GetModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the requested model. - dataset_id (str): - Required. Dataset ID of the requested model. - model_id (str): - Required. Model ID of the requested model. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -class PatchModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the model to patch. - dataset_id (str): - Required. Dataset ID of the model to patch. - model_id (str): - Required. Model ID of the model to patch. - model (google.cloud.bigquery_v2.types.Model): - Required. Patched model. - Follows RFC5789 patch semantics. Missing fields - are not updated. To clear a field, explicitly - set to default value. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - model = proto.Field(proto.MESSAGE, number=4, message="Model",) - - -class DeleteModelRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the model to delete. - dataset_id (str): - Required. Dataset ID of the model to delete. - model_id (str): - Required. Model ID of the model to delete. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -class ListModelsRequest(proto.Message): - r""" - Attributes: - project_id (str): - Required. Project ID of the models to list. - dataset_id (str): - Required. Dataset ID of the models to list. - max_results (google.protobuf.wrappers_pb2.UInt32Value): - The maximum number of results to return in a - single response page. Leverage the page tokens - to iterate through the entire collection. - page_token (str): - Page token, returned by a previous call to - request the next page of results - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - max_results = proto.Field( - proto.MESSAGE, number=3, message=wrappers_pb2.UInt32Value, - ) - page_token = proto.Field(proto.STRING, number=4,) - - -class ListModelsResponse(proto.Message): - r""" - Attributes: - models (Sequence[google.cloud.bigquery_v2.types.Model]): - Models in the requested dataset. Only the following fields - are populated: model_reference, model_type, creation_time, - last_modified_time and labels. - next_page_token (str): - A token to request the next page of results. - """ - - @property - def raw_page(self): - return self - - models = proto.RepeatedField(proto.MESSAGE, number=1, message="Model",) - next_page_token = proto.Field(proto.STRING, number=2,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/model_reference.py b/google/cloud/bigquery_v2/types/model_reference.py deleted file mode 100644 index a9ebad613..000000000 --- a/google/cloud/bigquery_v2/types/model_reference.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"ModelReference",}, -) - - -class ModelReference(proto.Message): - r"""Id path of a model. - Attributes: - project_id (str): - Required. The ID of the project containing - this model. - dataset_id (str): - Required. The ID of the dataset containing - this model. - model_id (str): - Required. The ID of the model. The ID must contain only - letters (a-z, A-Z), numbers (0-9), or underscores (_). The - maximum length is 1,024 characters. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - model_id = proto.Field(proto.STRING, number=3,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py deleted file mode 100644 index 7a845fc48..000000000 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", - manifest={ - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", - }, -) - - -class StandardSqlDataType(proto.Message): - r"""The type of a variable, e.g., a function argument. Examples: INT64: - {type_kind="INT64"} ARRAY: {type_kind="ARRAY", - array_element_type="STRING"} STRUCT: - {type_kind="STRUCT", struct_type={fields=[ {name="x", - type={type_kind="STRING"}}, {name="y", type={type_kind="ARRAY", - array_element_type="DATE"}} ]}} - - Attributes: - type_kind (google.cloud.bigquery_v2.types.StandardSqlDataType.TypeKind): - Required. The top level type of this field. - Can be any standard SQL data type (e.g., - "INT64", "DATE", "ARRAY"). - array_element_type (google.cloud.bigquery_v2.types.StandardSqlDataType): - The type of the array's elements, if type_kind = "ARRAY". - struct_type (google.cloud.bigquery_v2.types.StandardSqlStructType): - The fields of this struct, in order, if type_kind = - "STRUCT". - """ - - class TypeKind(proto.Enum): - r"""""" - TYPE_KIND_UNSPECIFIED = 0 - INT64 = 2 - BOOL = 5 - FLOAT64 = 7 - STRING = 8 - BYTES = 9 - TIMESTAMP = 19 - DATE = 10 - TIME = 20 - DATETIME = 21 - INTERVAL = 26 - GEOGRAPHY = 22 - NUMERIC = 23 - BIGNUMERIC = 24 - JSON = 25 - ARRAY = 16 - STRUCT = 17 - - type_kind = proto.Field(proto.ENUM, number=1, enum=TypeKind,) - array_element_type = proto.Field( - proto.MESSAGE, number=2, oneof="sub_type", message="StandardSqlDataType", - ) - struct_type = proto.Field( - proto.MESSAGE, number=3, oneof="sub_type", message="StandardSqlStructType", - ) - - -class StandardSqlField(proto.Message): - r"""A field or a column. - Attributes: - name (str): - Optional. The name of this field. Can be - absent for struct fields. - type (google.cloud.bigquery_v2.types.StandardSqlDataType): - Optional. The type of this parameter. Absent - if not explicitly specified (e.g., CREATE - FUNCTION statement can omit the return type; in - this case the output parameter does not have - this "type" field). - """ - - name = proto.Field(proto.STRING, number=1,) - type = proto.Field(proto.MESSAGE, number=2, message="StandardSqlDataType",) - - -class StandardSqlStructType(proto.Message): - r""" - Attributes: - fields (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - - """ - - fields = proto.RepeatedField(proto.MESSAGE, number=1, message="StandardSqlField",) - - -class StandardSqlTableType(proto.Message): - r"""A table type - Attributes: - columns (Sequence[google.cloud.bigquery_v2.types.StandardSqlField]): - The columns in this table type - """ - - columns = proto.RepeatedField(proto.MESSAGE, number=1, message="StandardSqlField",) - - -__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_v2/types/table_reference.py b/google/cloud/bigquery_v2/types/table_reference.py deleted file mode 100644 index d56e5b09f..000000000 --- a/google/cloud/bigquery_v2/types/table_reference.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import proto # type: ignore - - -__protobuf__ = proto.module( - package="google.cloud.bigquery.v2", manifest={"TableReference",}, -) - - -class TableReference(proto.Message): - r""" - Attributes: - project_id (str): - Required. The ID of the project containing - this table. - dataset_id (str): - Required. The ID of the dataset containing - this table. - table_id (str): - Required. The ID of the table. The ID must contain only - letters (a-z, A-Z), numbers (0-9), or underscores (_). The - maximum length is 1,024 characters. Certain operations allow - suffixing of the table ID with a partition decorator, such - as ``sample_table$20190123``. - project_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - dataset_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - table_id_alternative (Sequence[str]): - The alternative field that will be used when ESF is not able - to translate the received data to the project_id field. - """ - - project_id = proto.Field(proto.STRING, number=1,) - dataset_id = proto.Field(proto.STRING, number=2,) - table_id = proto.Field(proto.STRING, number=3,) - project_id_alternative = proto.RepeatedField(proto.STRING, number=4,) - dataset_id_alternative = proto.RepeatedField(proto.STRING, number=5,) - table_id_alternative = proto.RepeatedField(proto.STRING, number=6,) - - -__all__ = tuple(sorted(__protobuf__.manifest)) From f57844f9117dfe326b2bef69cfd191d8b7d16a31 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 6 Aug 2021 16:26:32 +0200 Subject: [PATCH 02/32] Replace proto Sql type classes with manual ones --- google/cloud/bigquery/enums.py | 22 +--- google/cloud/bigquery/model.py | 2 +- google/cloud/bigquery/routine/routine.py | 14 +-- google/cloud/bigquery/schema.py | 21 ++-- google/cloud/bigquery/types/__init__.py | 30 +++++ google/cloud/bigquery/types/standard_sql.py | 115 ++++++++++++++++++ setup.py | 1 + tests/system/test_client.py | 15 ++- .../enums/test_standard_sql_data_types.py | 2 +- tests/unit/model/test_model.py | 2 +- tests/unit/routine/test_routine.py | 29 +++-- tests/unit/routine/test_routine_argument.py | 14 +-- tests/unit/test_client.py | 6 +- tests/unit/test_schema.py | 44 ++++--- 14 files changed, 228 insertions(+), 89 deletions(-) create mode 100644 google/cloud/bigquery/types/__init__.py create mode 100644 google/cloud/bigquery/types/standard_sql.py diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index d67cebd4c..5f001d8c4 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -12,12 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - import enum -import itertools -from google.cloud.bigquery_v2 import types as gapic_types +from google.cloud.bigquery import types from google.cloud.bigquery.query import ScalarQueryParameterType @@ -203,28 +200,17 @@ class KeyResultStatementKind: def _make_sql_scalars_enum(): - """Create an enum based on a gapic enum containing only SQL scalar types.""" + """Create an enum based on the types enum containing only SQL scalar types.""" new_enum = enum.Enum( "StandardSqlDataTypes", ( (member.name, member.value) - for member in gapic_types.StandardSqlDataType.TypeKind + for member in types.StandardSqlDataType.TypeKind if member.name in _SQL_SCALAR_TYPES ), ) - - # make sure the docstring for the new enum is also correct - orig_doc = gapic_types.StandardSqlDataType.TypeKind.__doc__ - skip_pattern = re.compile( - "|".join(_SQL_NONSCALAR_TYPES) - + "|because a JSON object" # the second description line of STRUCT member - ) - - new_doc = "\n".join( - itertools.filterfalse(skip_pattern.search, orig_doc.splitlines()) - ) - new_enum.__doc__ = "An Enum of scalar SQL types.\n" + new_doc + new_enum.__doc__ = types.StandardSqlDataType.__doc__ return new_enum diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 2d3f6660f..31ac15907 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -23,7 +23,7 @@ import google.cloud._helpers from google.api_core import datetime_helpers from google.cloud.bigquery import _helpers -from google.cloud.bigquery_v2 import types +from google.cloud.bigquery import types from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index a776212c3..3956db9c3 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -20,8 +20,8 @@ import google.cloud._helpers from google.cloud.bigquery import _helpers -import google.cloud.bigquery_v2.types -from google.cloud.bigquery_v2.types import StandardSqlTableType +from google.cloud.bigquery.types import StandardSqlDataType +from google.cloud.bigquery.types import StandardSqlTableType class RoutineType: @@ -190,7 +190,7 @@ def arguments(self, value): @property def return_type(self): - """google.cloud.bigquery_v2.types.StandardSqlDataType: Return type of + """google.cloud.bigquery.types.StandardSqlDataType: Return type of the routine. If absent, the return type is inferred from @@ -206,7 +206,7 @@ def return_type(self): if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlDataType() + output = StandardSqlDataType() raw_protobuf = json_format.ParseDict( resource, output._pb, ignore_unknown_fields=True ) @@ -232,7 +232,7 @@ def return_table_type(self) -> StandardSqlTableType: if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlTableType() + output = StandardSqlTableType() raw_protobuf = json_format.ParseDict( resource, output._pb, ignore_unknown_fields=True ) @@ -407,7 +407,7 @@ def mode(self, value): @property def data_type(self): - """Optional[google.cloud.bigquery_v2.types.StandardSqlDataType]: Type + """Optional[google.cloud.bigquery.types.StandardSqlDataType]: Type of a variable, e.g., a function argument. See: @@ -417,7 +417,7 @@ def data_type(self): if not resource: return resource - output = google.cloud.bigquery_v2.types.StandardSqlDataType() + output = StandardSqlDataType() raw_protobuf = json_format.ParseDict( resource, output._pb, ignore_unknown_fields=True ) diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 157db7ce6..7ae3a7d05 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -17,7 +17,7 @@ import collections from typing import Optional -from google.cloud.bigquery_v2 import types +from google.cloud.bigquery import types _DEFAULT_VALUE = object() @@ -286,11 +286,7 @@ def _key(self): ) def to_standard_sql(self) -> types.StandardSqlField: - """Return the field as the standard SQL field representation object. - - Returns: - An instance of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. - """ + """Return the field as the standard SQL field representation object.""" sql_type = types.StandardSqlDataType() if self.mode == "REPEATED": @@ -306,7 +302,9 @@ def to_standard_sql(self) -> types.StandardSqlField: self.field_type, types.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, ) - sql_type.array_element_type.type_kind = array_element_type + sql_type.array_element_type = types.StandardSqlDataType( + type_kind=array_element_type + ) # ARRAY cannot directly contain other arrays, only scalar types and STRUCTs # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type @@ -314,16 +312,15 @@ def to_standard_sql(self) -> types.StandardSqlField: array_element_type == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 ): - sql_type.array_element_type.struct_type.fields.extend( - field.to_standard_sql() for field in self.fields + sql_type.array_element_type.struct_type = types.StandardSqlStructType( + fields=(field.to_standard_sql() for field in self.fields) ) - elif ( sql_type.type_kind == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 ): - sql_type.struct_type.fields.extend( - field.to_standard_sql() for field in self.fields + sql_type.struct_type = types.StandardSqlStructType( + fields=(field.to_standard_sql() for field in self.fields) ) return types.StandardSqlField(name=self.name, type=sql_type) diff --git a/google/cloud/bigquery/types/__init__.py b/google/cloud/bigquery/types/__init__.py new file mode 100644 index 000000000..43f3e40bf --- /dev/null +++ b/google/cloud/bigquery/types/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: This subpackage define type classes that were previously generated from protobuf +# definitions in library versions before 3.0. + +from .standard_sql import ( + StandardSqlDataType, + StandardSqlField, + StandardSqlStructType, + StandardSqlTableType, +) + +__all__ = ( + "StandardSqlDataType", + "StandardSqlField", + "StandardSqlStructType", + "StandardSqlTableType", +) diff --git a/google/cloud/bigquery/types/standard_sql.py b/google/cloud/bigquery/types/standard_sql.py new file mode 100644 index 000000000..121d0617f --- /dev/null +++ b/google/cloud/bigquery/types/standard_sql.py @@ -0,0 +1,115 @@ +# Copyright 2021 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +import enum +from typing import Iterable, Optional + + +@dataclass +class StandardSqlDataType: + """The type of a variable, e.g., a function argument. + + Examples: + + INT64: {type_kind="INT64"} + ARRAY: {type_kind="ARRAY", array_element_type="STRING"} + STRUCT: { + type_kind="STRUCT", + struct_type={ + fields=[ + {name="x", type={type_kind="STRING"}}, + { + name="y", + type={type_kind="ARRAY", array_element_type="DATE"} + } + ] + } + } + """ + + class TypeKind(str, enum.Enum): + def _generate_next_value_(name, start, count, last_values): + return name + + TYPE_KIND_UNSPECIFIED = enum.auto() + INT64 = enum.auto() + BOOL = enum.auto() + FLOAT64 = enum.auto() + STRING = enum.auto() + BYTES = enum.auto() + TIMESTAMP = enum.auto() + DATE = enum.auto() + TIME = enum.auto() + DATETIME = enum.auto() + INTERVAL = enum.auto() + GEOGRAPHY = enum.auto() + NUMERIC = enum.auto() + BIGNUMERIC = enum.auto() + JSON = enum.auto() + ARRAY = enum.auto() + STRUCT = enum.auto() + + type_kind: Optional[TypeKind] = TypeKind.TYPE_KIND_UNSPECIFIED + """The top level type of this field. Can be any standard SQL data type, + e.g. INT64, DATE, ARRAY. + """ + + array_element_type: Optional["StandardSqlDataType"] = None + """The type of the array's elements, if type_kind is ARRAY.""" + + struct_type: Optional["StandardSqlStructType"] = None + """The fields of this struct, in order, if type_kind is STRUCT.""" + + def __post_init__(self): + if self.array_element_type is not None and self.struct_type is not None: + raise ValueError( + "array_element_type and struct_type are mutally exclusive." + ) + + +@dataclass +class StandardSqlField: + """A field or a column.""" + + name: Optional[str] = None + """The name of this field. Can be absent for struct fields.""" + + type: Optional["StandardSqlDataType"] = None + """The type of this parameter. Absent if not explicitly specified. + + For example, CREATE FUNCTION statement can omit the return type; in this case the + output parameter does not have this "type" field). + """ + + +@dataclass(init=False) +class StandardSqlStructType: + """Type of a struct field.""" + + fields: Optional[Iterable["StandardSqlField"]] + + def __init__(self, fields=None): + self.fields = [] if fields is None else [field for field in fields] + + +@dataclass(init=False) +class StandardSqlTableType: + """A table type""" + + columns: Iterable[StandardSqlField] + """The columns in this table type""" + + def __init__(self, columns): + self.columns = [col for col in columns] diff --git a/setup.py b/setup.py index 6fa619d37..bac711e4b 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ + "dataclasses >= 0.8, < 1.0; python_version < '3.7'", # Until we drop Python 3.6 support. "grpcio >= 1.38.1, < 2.0dev", # https://github.com/googleapis/python-bigquery/issues/695 # NOTE: Maintainers, please do not require google-api-core>=2.x.x # Until this issue is closed diff --git a/tests/system/test_client.py b/tests/system/test_client.py index c6896da14..894b28d58 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -51,7 +51,6 @@ from google.api_core.exceptions import TooManyRequests from google.api_core.iam import Policy from google.cloud import bigquery -from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference from google.cloud.bigquery.table import Table @@ -2181,8 +2180,8 @@ def test_insert_rows_nested_nested_dictionary(self): def test_create_routine(self): routine_name = "test_routine" dataset = self.temp_dataset(_make_dataset_id("create_routine")) - float64_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.FLOAT64 + float64_type = bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.FLOAT64 ) routine = bigquery.Routine( dataset.routine(routine_name), @@ -2196,8 +2195,8 @@ def test_create_routine(self): routine.arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.ARRAY, + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.ARRAY, array_element_type=float64_type, ), ) @@ -2218,9 +2217,9 @@ def test_create_routine(self): def test_create_tvf_routine(self): from google.cloud.bigquery import Routine, RoutineArgument, RoutineType - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.types.StandardSqlDataType + StandardSqlField = bigquery.types.StandardSqlField + StandardSqlTableType = bigquery.types.StandardSqlTableType INT64 = StandardSqlDataType.TypeKind.INT64 STRING = StandardSqlDataType.TypeKind.STRING diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py index 7f62c46fd..d3c08a099 100644 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ b/tests/unit/enums/test_standard_sql_data_types.py @@ -32,7 +32,7 @@ def enum_under_test(): @pytest.fixture def gapic_enum(): """The referential autogenerated enum the enum under test is based on.""" - from google.cloud.bigquery_v2.types import StandardSqlDataType + from google.cloud.bigquery.types import StandardSqlDataType return StandardSqlDataType.TypeKind diff --git a/tests/unit/model/test_model.py b/tests/unit/model/test_model.py index 8f0bf58d5..0f264ba00 100644 --- a/tests/unit/model/test_model.py +++ b/tests/unit/model/test_model.py @@ -19,7 +19,7 @@ import pytest import google.cloud._helpers -from google.cloud.bigquery_v2 import types +from google.cloud.bigquery import types KMS_KEY_NAME = "projects/1/locations/us/keyRings/1/cryptoKeys/1" diff --git a/tests/unit/routine/test_routine.py b/tests/unit/routine/test_routine.py index fdaf13324..66b05d2fb 100644 --- a/tests/unit/routine/test_routine.py +++ b/tests/unit/routine/test_routine.py @@ -19,7 +19,6 @@ import google.cloud._helpers from google.cloud import bigquery -from google.cloud import bigquery_v2 @pytest.fixture @@ -62,15 +61,15 @@ def test_ctor_w_properties(target_class): arguments = [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ) ] body = "x * 3" language = "SQL" - return_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + return_type = bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ) type_ = "SCALAR_FUNCTION" description = "A routine description." @@ -146,15 +145,15 @@ def test_from_api_repr(target_class): assert actual_routine.arguments == [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ) ] assert actual_routine.body == "42" assert actual_routine.language == "SQL" - assert actual_routine.return_type == bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + assert actual_routine.return_type == bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ) assert actual_routine.return_table_type is None assert actual_routine.type_ == "SCALAR_FUNCTION" @@ -168,9 +167,9 @@ def test_from_api_repr_tvf_function(target_class): from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.routine import RoutineType - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.types.StandardSqlDataType + StandardSqlField = bigquery.types.StandardSqlField + StandardSqlTableType = bigquery.types.StandardSqlTableType creation_time = datetime.datetime( 2010, 5, 19, 16, 0, 0, tzinfo=google.cloud._helpers.UTC @@ -460,9 +459,9 @@ def test_set_return_table_type_w_none(object_under_test): def test_set_return_table_type_w_not_none(object_under_test): - StandardSqlDataType = bigquery_v2.types.StandardSqlDataType - StandardSqlField = bigquery_v2.types.StandardSqlField - StandardSqlTableType = bigquery_v2.types.StandardSqlTableType + StandardSqlDataType = bigquery.types.StandardSqlDataType + StandardSqlField = bigquery.types.StandardSqlField + StandardSqlTableType = bigquery.types.StandardSqlTableType table_type = StandardSqlTableType( columns=[ diff --git a/tests/unit/routine/test_routine_argument.py b/tests/unit/routine/test_routine_argument.py index e3bda9539..d30830444 100644 --- a/tests/unit/routine/test_routine_argument.py +++ b/tests/unit/routine/test_routine_argument.py @@ -16,7 +16,7 @@ import pytest -from google.cloud import bigquery_v2 +from google.cloud import bigquery @pytest.fixture @@ -27,8 +27,8 @@ def target_class(): def test_ctor(target_class): - data_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ) actual_arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type @@ -50,8 +50,8 @@ def test_from_api_repr(target_class): assert actual_arg.name == "field_name" assert actual_arg.kind == "FIXED_TYPE" assert actual_arg.mode == "IN" - assert actual_arg.data_type == bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + assert actual_arg.data_type == bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ) @@ -71,8 +71,8 @@ def test_from_api_repr_w_unknown_fields(target_class): def test_eq(target_class): - data_type = bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ) arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index bd07990b8..fbe22067d 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -49,8 +49,8 @@ import google.api_core.exceptions from google.api_core import client_info import google.cloud._helpers +from google.cloud import bigquery from google.cloud import bigquery_storage -from google.cloud import bigquery_v2 from google.cloud.bigquery.dataset import DatasetReference from tests.unit.helpers import make_connection @@ -1859,8 +1859,8 @@ def test_update_routine(self): routine.arguments = [ RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ) ] diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index d0b5ca54c..8ff70b595 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.cloud.bigquery.types.standard_sql import StandardSqlStructType from google.cloud.bigquery.schema import PolicyTagList import unittest @@ -28,7 +29,7 @@ def _get_target_class(): @staticmethod def _get_standard_sql_data_type_class(): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import types return types.StandardSqlDataType @@ -224,7 +225,7 @@ def test_to_standard_sql_simple_type(self): self.assertEqual(standard_field.type.type_kind, standard_type) def test_to_standard_sql_struct_type(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import types # Expected result object: # @@ -267,10 +268,13 @@ def test_to_standard_sql_struct_type(self): # level 1 fields sub_field_struct = types.StandardSqlField( - name="last_used", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) - ) - sub_field_struct.type.struct_type.fields.extend( - [sub_sub_field_date, sub_sub_field_time] + name="last_used", + type=sql_type( + type_kind=sql_type.TypeKind.STRUCT, + struct_type=types.StandardSqlStructType( + fields=[sub_sub_field_date, sub_sub_field_time] + ), + ), ) sub_field_bytes = types.StandardSqlField( name="image_content", type=sql_type(type_kind=sql_type.TypeKind.BYTES) @@ -278,10 +282,13 @@ def test_to_standard_sql_struct_type(self): # level 0 (top level) expected_result = types.StandardSqlField( - name="image_usage", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) - ) - expected_result.type.struct_type.fields.extend( - [sub_field_bytes, sub_field_struct] + name="image_usage", + type=sql_type( + type_kind=sql_type.TypeKind.STRUCT, + struct_type=types.StandardSqlStructType( + fields=[sub_field_bytes, sub_field_struct] + ), + ), ) # construct legacy SchemaField object @@ -300,13 +307,15 @@ def test_to_standard_sql_struct_type(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_simple(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import types sql_type = self._get_standard_sql_data_type_class() # construct expected result object - expected_sql_type = sql_type(type_kind=sql_type.TypeKind.ARRAY) - expected_sql_type.array_element_type.type_kind = sql_type.TypeKind.INT64 + expected_sql_type = sql_type( + type_kind=sql_type.TypeKind.ARRAY, + array_element_type=sql_type(type_kind=sql_type.TypeKind.INT64), + ) expected_result = types.StandardSqlField( name="valid_numbers", type=expected_sql_type ) @@ -318,7 +327,7 @@ def test_to_standard_sql_array_type_simple(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_struct(self): - from google.cloud.bigquery_v2 import types + from google.cloud.bigquery import types sql_type = self._get_standard_sql_data_type_class() @@ -330,9 +339,12 @@ def test_to_standard_sql_array_type_struct(self): name="age", type=sql_type(type_kind=sql_type.TypeKind.INT64) ) person_struct = types.StandardSqlField( - name="person_info", type=sql_type(type_kind=sql_type.TypeKind.STRUCT) + name="person_info", + type=sql_type( + type_kind=sql_type.TypeKind.STRUCT, + struct_type=StandardSqlStructType(fields=[name_field, age_field]), + ), ) - person_struct.type.struct_type.fields.extend([name_field, age_field]) # define expected result - an ARRAY of person structs expected_sql_type = sql_type( From a2011880764e8bf7abccd1d02bb50ddfd521b330 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 19 Aug 2021 14:28:31 +0200 Subject: [PATCH 03/32] Enhance standard Sql model classes --- google/cloud/bigquery/types/standard_sql.py | 145 +++++++- tests/unit/test_standard_sql_types.py | 378 ++++++++++++++++++++ 2 files changed, 505 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_standard_sql_types.py diff --git a/google/cloud/bigquery/types/standard_sql.py b/google/cloud/bigquery/types/standard_sql.py index 121d0617f..87966ebe2 100644 --- a/google/cloud/bigquery/types/standard_sql.py +++ b/google/cloud/bigquery/types/standard_sql.py @@ -14,29 +14,34 @@ from dataclasses import dataclass import enum -from typing import Iterable, Optional +from typing import Any, Dict, Iterable, Optional @dataclass class StandardSqlDataType: """The type of a variable, e.g., a function argument. + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType + Examples: - INT64: {type_kind="INT64"} - ARRAY: {type_kind="ARRAY", array_element_type="STRING"} - STRUCT: { - type_kind="STRUCT", - struct_type={ - fields=[ - {name="x", type={type_kind="STRING"}}, - { - name="y", - type={type_kind="ARRAY", array_element_type="DATE"} - } - ] + .. code-block:: text + + INT64: {type_kind="INT64"} + ARRAY: {type_kind="ARRAY", array_element_type="STRING"} + STRUCT: { + type_kind="STRUCT", + struct_type={ + fields=[ + {name="x", type={type_kind="STRING"}}, + { + name="y", + type={type_kind="ARRAY", array_element_type="DATE"} + } + ] + } } - } """ class TypeKind(str, enum.Enum): @@ -78,10 +83,61 @@ def __post_init__(self): "array_element_type and struct_type are mutally exclusive." ) + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL data type.""" + + if not self.type_kind: + type_kind = self.TypeKind.TYPE_KIND_UNSPECIFIED.value + else: + type_kind = self.type_kind.value + + result = {"typeKind": type_kind} + + if self.type_kind == self.TypeKind.ARRAY: + if not self.array_element_type: + array_type = None + else: + array_type = self.array_element_type.to_api_repr() + result["arrayElementType"] = array_type + elif self.type_kind == self.TypeKind.STRUCT: + if not self.struct_type: + struct_type = None + else: + struct_type = self.struct_type.to_api_repr() + result["structType"] = struct_type + + return result + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]): + """Construct an SQL data type instance given its API representation.""" + type_kind = resource.get("typeKind") + if type_kind not in cls.TypeKind.__members__: + type_kind = cls.TypeKind.TYPE_KIND_UNSPECIFIED + else: + # Convert string to an enum member. + type_kind = cls.TypeKind[type_kind] # pytype: disable=missing-parameter + + array_element_type = None + if type_kind == cls.type_kind.ARRAY: + element_type = resource.get("arrayElementType", {}) + array_element_type = cls.from_api_repr(element_type) + + struct_type = None + if type_kind == cls.TypeKind.STRUCT: + struct_info = resource.get("structType", {}) + struct_type = StandardSqlStructType.from_api_repr(struct_info) + + return cls(type_kind, array_element_type, struct_type) + @dataclass class StandardSqlField: - """A field or a column.""" + """A field or a column. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlField + """ name: Optional[str] = None """The name of this field. Can be absent for struct fields.""" @@ -93,23 +149,76 @@ class StandardSqlField: output parameter does not have this "type" field). """ + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL field.""" + type_repr = None if self.type is None else self.type.to_api_repr() + return {"name": self.name, "type": type_repr} + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]): + """Construct an SQL field instance given its API representation.""" + result = cls( + name=resource.get("name"), + type=StandardSqlDataType.from_api_repr(resource.get("type", {})), + ) + return result + @dataclass(init=False) class StandardSqlStructType: - """Type of a struct field.""" + """Type of a struct field. + + See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType#StandardSqlStructType + """ fields: Optional[Iterable["StandardSqlField"]] def __init__(self, fields=None): self.fields = [] if fields is None else [field for field in fields] + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL struct type.""" + fields = [field.to_api_repr() for field in self.fields] + result = {"fields": fields} + return result + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlStructType": + """Construct an SQL struct type instance given its API representation.""" + fields = ( + StandardSqlField.from_api_repr(field) + for field in resource.get("fields", []) + ) + return cls(fields=fields) + @dataclass(init=False) class StandardSqlTableType: - """A table type""" + """A table type. + + https://cloud.google.com/workflows/docs/reference/googleapis/bigquery/v2/Overview#StandardSqlTableType + """ columns: Iterable[StandardSqlField] """The columns in this table type""" - def __init__(self, columns): + def __init__(self, columns: Iterable[StandardSqlField]): self.columns = [col for col in columns] + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL table type.""" + columns = [col.to_api_repr() for col in self.columns] + return {"columns": columns} + + @classmethod + def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlTableType": + """Construct an SQL table type instance given its API representation.""" + columns = ( + StandardSqlField( + name=column.get("name"), + type=StandardSqlDataType.from_api_repr(column.get("type", {})), + ) + for column in resource.get("columns", []) + ) + return cls(columns=columns) diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py new file mode 100644 index 000000000..ea65800eb --- /dev/null +++ b/tests/unit/test_standard_sql_types.py @@ -0,0 +1,378 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +class TestStandardSqlDataType: + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.types.standard_sql import StandardSqlDataType + + return StandardSqlDataType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor_w_both_array_element_and_struct_types(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + + StandardSqlDataType = self._get_target_class() + TypeKind = StandardSqlDataType.TypeKind + + array_element_type = StandardSqlDataType(TypeKind.INT64) + struct_type = StandardSqlStructType( + fields=[ + StandardSqlField("name", StandardSqlDataType(TypeKind.STRING)), + StandardSqlField("age", StandardSqlDataType(TypeKind.INT64)), + ] + ) + + with pytest.raises(ValueError, match=r".*mutally exclusive.*"): + self._make_one( + type_kind=TypeKind.TYPE_KIND_UNSPECIFIED, + array_element_type=array_element_type, + struct_type=struct_type, + ) + + def test_ctor_default_type_kind(self): + TypeKind = self._get_target_class().TypeKind + instance = self._make_one() + assert instance.type_kind == TypeKind.TYPE_KIND_UNSPECIFIED + + def test_to_api_repr_no_type_set(self): + instance = self._make_one() + instance.type_kind = None + + result = instance.to_api_repr() + + assert result == {"typeKind": "TYPE_KIND_UNSPECIFIED"} + + def test_to_api_repr_scalar_type(self): + TypeKind = self._get_target_class().TypeKind + instance = self._make_one(TypeKind.FLOAT64) + + result = instance.to_api_repr() + + assert result == {"typeKind": "FLOAT64"} + + def test_to_api_repr_array_type_element_type_missing(self): + TypeKind = self._get_target_class().TypeKind + instance = self._make_one(TypeKind.ARRAY, array_element_type=None) + + result = instance.to_api_repr() + + expected = {"typeKind": "ARRAY", "arrayElementType": None} + assert result == expected + + def test_to_api_repr_array_type_w_element_type(self): + TypeKind = self._get_target_class().TypeKind + + array_element_type = self._make_one(type_kind=TypeKind.BOOL) + instance = self._make_one(TypeKind.ARRAY, array_element_type=array_element_type) + + result = instance.to_api_repr() + + expected = {"typeKind": "ARRAY", "arrayElementType": {"typeKind": "BOOL"}} + assert result == expected + + def test_to_api_repr_struct_type_field_types_missing(self): + TypeKind = self._get_target_class().TypeKind + instance = self._make_one(TypeKind.STRUCT, struct_type=None) + + result = instance.to_api_repr() + + assert result == {"typeKind": "STRUCT", "structType": None} + + def test_to_api_repr_struct_type_w_field_types(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + + StandardSqlDataType = self._get_target_class() + TypeKind = StandardSqlDataType.TypeKind + + person_type = StandardSqlStructType( + fields=[ + StandardSqlField("name", StandardSqlDataType(TypeKind.STRING)), + StandardSqlField("age", StandardSqlDataType(TypeKind.INT64)), + ] + ) + employee_type = StandardSqlStructType( + fields=[ + StandardSqlField("job_title", StandardSqlDataType(TypeKind.STRING)), + StandardSqlField("salary", StandardSqlDataType(TypeKind.FLOAT64)), + StandardSqlField( + "employee_info", + StandardSqlDataType( + type_kind=TypeKind.STRUCT, struct_type=person_type, + ), + ), + ] + ) + + instance = self._make_one(TypeKind.STRUCT, struct_type=employee_type) + result = instance.to_api_repr() + + expected = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "job_title", "type": {"typeKind": "STRING"}}, + {"name": "salary", "type": {"typeKind": "FLOAT64"}}, + { + "name": "employee_info", + "type": { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "name", "type": {"typeKind": "STRING"}}, + {"name": "age", "type": {"typeKind": "INT64"}}, + ], + }, + }, + }, + ], + }, + } + assert result == expected + + def test_from_api_repr_empty_resource(self): + klass = self._get_target_class() + result = klass.from_api_repr(resource={}) + + expected = klass( + type_kind=klass.TypeKind.TYPE_KIND_UNSPECIFIED, + array_element_type=None, + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_scalar_type(self): + klass = self._get_target_class() + resource = {"typeKind": "DATE"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=klass.TypeKind.DATE, array_element_type=None, struct_type=None, + ) + assert result == expected + + def test_from_api_repr_array_type_full(self): + klass = self._get_target_class() + resource = {"typeKind": "ARRAY", "arrayElementType": {"typeKind": "BYTES"}} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=klass.TypeKind.ARRAY, + array_element_type=klass(type_kind=klass.TypeKind.BYTES), + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_array_type_missing_element_type(self): + klass = self._get_target_class() + resource = {"typeKind": "ARRAY"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=klass.TypeKind.ARRAY, + array_element_type=klass( + type_kind=klass.TypeKind.TYPE_KIND_UNSPECIFIED, + array_element_type=None, + struct_type=None, + ), + struct_type=None, + ) + assert result == expected + + def test_from_api_repr_struct_type_nested(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + + klass = self._get_target_class() + TypeKind = klass.TypeKind + + resource = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "job_title", "type": {"typeKind": "STRING"}}, + {"name": "salary", "type": {"typeKind": "FLOAT64"}}, + { + "name": "employee_info", + "type": { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"name": "name", "type": {"typeKind": "STRING"}}, + {"name": "age", "type": {"typeKind": "INT64"}}, + ], + }, + }, + }, + ], + }, + } + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=TypeKind.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField("job_title", klass(TypeKind.STRING)), + StandardSqlField("salary", klass(TypeKind.FLOAT64)), + StandardSqlField( + "employee_info", + klass( + type_kind=TypeKind.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField("name", klass(TypeKind.STRING)), + StandardSqlField("age", klass(TypeKind.INT64)), + ] + ), + ), + ), + ] + ), + ) + assert result == expected + + def test_from_api_repr_struct_type_missing_struct_info(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + + klass = self._get_target_class() + resource = {"typeKind": "STRUCT"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=klass.TypeKind.STRUCT, + array_element_type=None, + struct_type=StandardSqlStructType(fields=[]), + ) + assert result == expected + + def test_from_api_repr_struct_type_incomplete_field_info(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + + klass = self._get_target_class() + TypeKind = klass.TypeKind + + resource = { + "typeKind": "STRUCT", + "structType": { + "fields": [ + {"type": {"typeKind": "STRING"}}, # missing name + {"name": "salary"}, # missing type + ], + }, + } + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=TypeKind.STRUCT, + struct_type=StandardSqlStructType( + fields=[ + StandardSqlField(None, klass(TypeKind.STRING)), + StandardSqlField("salary", klass(TypeKind.TYPE_KIND_UNSPECIFIED)), + ] + ), + ) + assert result == expected + + +class TestStandardSqlTableType: + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.types.standard_sql import StandardSqlTableType + + return StandardSqlTableType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_columns_shallow_copy(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + + columns = [ + StandardSqlField("foo"), + StandardSqlField("bar"), + StandardSqlField("baz"), + ] + + instance = self._make_one(columns=columns) + + assert len(instance.columns) == 3 + columns.pop() + assert len(instance.columns) == 3 # Still the same. + + def test_to_api_repr_no_columns(self): + instance = self._make_one(columns=[]) + result = instance.to_api_repr() + assert result == {"columns": []} + + def test_to_api_repr_with_columns(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlField + + columns = [StandardSqlField("foo"), StandardSqlField("bar")] + instance = self._make_one(columns=columns) + + result = instance.to_api_repr() + + expected = { + "columns": [{"name": "foo", "type": None}, {"name": "bar", "type": None}] + } + assert result == expected + + def test_from_api_repr_missing_columns(self): + resource = {} + result = self._get_target_class().from_api_repr(resource) + assert result.columns == [] + + def test_from_api_repr_with_incomplete_columns(self): + from google.cloud.bigquery.types.standard_sql import StandardSqlDataType + from google.cloud.bigquery.types.standard_sql import StandardSqlField + + resource = { + "columns": [ + {"type": {"typeKind": "BOOL"}}, # missing name + {"name": "bar"}, # missing type + ] + } + + result = self._get_target_class().from_api_repr(resource) + + assert len(result.columns) == 2 + + expected = StandardSqlField( + name=None, + type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.BOOL), + ) + assert result.columns[0] == expected + + expected = StandardSqlField( + name="bar", + type=StandardSqlDataType( + type_kind=StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED + ), + ) + assert result.columns[1] == expected From e2089b310b5171dd243b85a668af365af5ab195a Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 12 Aug 2021 13:16:17 +0200 Subject: [PATCH 04/32] Adjust models code to proto classes removal --- google/cloud/bigquery/model.py | 247 +++++++++++++-------------------- tests/unit/model/test_model.py | 56 ++++---- tests/unit/test_client.py | 2 +- 3 files changed, 126 insertions(+), 179 deletions(-) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 31ac15907..21c2417fe 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -17,8 +17,8 @@ """Define resources for the BigQuery ML Models API.""" import copy - -from google.protobuf import json_format +import datetime +from typing import Any, Dict, Optional, Sequence, Union import google.cloud._helpers from google.api_core import datetime_helpers @@ -27,14 +27,14 @@ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -class Model(object): +class Model: """Model represents a machine learning model resource. See https://cloud.google.com/bigquery/docs/reference/rest/v2/models Args: - model_ref (Union[google.cloud.bigquery.model.ModelReference, str]): + model_ref: A pointer to a model. If ``model_ref`` is a string, it must included a project ID, dataset ID, and model ID, each separated by ``.``. @@ -51,11 +51,7 @@ class Model(object): "encryption_configuration": "encryptionConfiguration", } - def __init__(self, model_ref): - # Use _proto on read-only properties to use it's built-in type - # conversion. - self._proto = types.Model()._pb - + def __init__(self, model_ref: Union["ModelReference", str, None]): # Use _properties on read-write properties to match the REST API # semantics. The BigQuery API makes a distinction between an unset # value, a null value, and a default value (0 or ""), but the protocol @@ -66,68 +62,63 @@ def __init__(self, model_ref): model_ref = ModelReference.from_string(model_ref) if model_ref: - self._proto.model_reference.CopyFrom(model_ref._proto) + self._properties["modelReference"] = model_ref.to_api_repr() @property - def reference(self): - """A :class:`~google.cloud.bigquery.model.ModelReference` pointing to - this model. + def reference(self) -> Optional["ModelReference"]: + """A model reference pointing to this model. Read-only. - - Returns: - google.cloud.bigquery.model.ModelReference: pointer to this model. """ - ref = ModelReference() - ref._proto = self._proto.model_reference - return ref + resource = self._properties.get("modelReference") + if resource is not None: + return ModelReference.from_api_repr(resource) @property - def project(self): - """str: Project bound to the model""" + def project(self) -> str: + """Project bound to the model.""" return self.reference.project @property - def dataset_id(self): - """str: ID of dataset containing the model.""" + def dataset_id(self) -> str: + """ID of dataset containing the model.""" return self.reference.dataset_id @property - def model_id(self): - """str: The model ID.""" + def model_id(self) -> str: + """The model ID.""" return self.reference.model_id @property - def path(self): - """str: URL path for the model's APIs.""" + def path(self) -> str: + """URL path for the model's APIs.""" return self.reference.path @property - def location(self): - """str: The geographic location where the model resides. This value - is inherited from the dataset. + def location(self) -> str: + """The geographic location where the model resides. + + This value is inherited from the dataset. Read-only. """ - return self._proto.location + return self._properties.get("location") @property - def etag(self): - """str: ETag for the model resource (:data:`None` until - set from the server). + def etag(self) -> str: + """ETag for the model resource (:data:`None` until set from the server). Read-only. """ - return self._proto.etag + return self._properties.get("etag") @property - def created(self): - """Union[datetime.datetime, None]: Datetime at which the model was - created (:data:`None` until set from the server). + def created(self) -> Optional[datetime.datetime]: + """Datetime at which the model was created (:data:`None` until set from the server). Read-only. """ - value = self._proto.creation_time + value = self._properties.get("creationTime") if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( @@ -135,13 +126,12 @@ def created(self): ) @property - def modified(self): - """Union[datetime.datetime, None]: Datetime at which the model was last - modified (:data:`None` until set from the server). + def modified(self) -> Optional[datetime.datetime]: + """Datetime at which the model was last modified (:data:`None` until set from the server). Read-only. """ - value = self._proto.last_modified_time + value = value = self._properties.get("lastModifiedTime") if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( @@ -149,57 +139,45 @@ def modified(self): ) @property - def model_type(self): - """google.cloud.bigquery_v2.types.Model.ModelType: Type of the - model resource. + def model_type(self) -> str: + """Type of the model resource. Read-only. - - The value is one of elements of the - :class:`~google.cloud.bigquery_v2.types.Model.ModelType` - enumeration. """ - return self._proto.model_type + return self._properties.get("modelType", "MODEL_TYPE_UNSPECIFIED") @property - def training_runs(self): - """Sequence[google.cloud.bigquery_v2.types.Model.TrainingRun]: Information - for all training runs in increasing order of start time. + def training_runs(self) -> Sequence[Dict[str, Any]]: + """Information for all training runs in increasing order of start time. Read-only. - - An iterable of :class:`~google.cloud.bigquery_v2.types.Model.TrainingRun`. """ - return self._proto.training_runs + return self._properties.get("trainingRuns", []) @property - def feature_columns(self): - """Sequence[google.cloud.bigquery_v2.types.StandardSqlField]: Input - feature columns that were used to train this model. + def feature_columns(self) -> Sequence[types.StandardSqlField]: + """Input feature columns that were used to train this model. Read-only. - - An iterable of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. """ - return self._proto.feature_columns + return self._properties.get("featureColumns", []) @property - def label_columns(self): - """Sequence[google.cloud.bigquery_v2.types.StandardSqlField]: Label - columns that were used to train this model. The output of the model - will have a ``predicted_`` prefix to these columns. + def label_columns(self) -> Sequence[types.StandardSqlField]: + """Label columns that were used to train this model. - Read-only. + The output of the model will have a ``predicted_`` prefix to these columns. - An iterable of :class:`~google.cloud.bigquery_v2.types.StandardSqlField`. + Read-only. """ - return self._proto.label_columns + return self._properties.get("labelColumns", []) @property - def expires(self): - """Union[datetime.datetime, None]: The datetime when this model - expires. If not present, the model will persist indefinitely. Expired - models will be deleted and their storage reclaimed. + def expires(self) -> Optional[datetime.datetime]: + """The datetime when this model expires. + + If not present, the model will persist indefinitely. Expired models will be + deleted and their storage reclaimed. """ value = self._properties.get("expirationTime") if value is not None: @@ -209,55 +187,48 @@ def expires(self): ) @expires.setter - def expires(self, value): + def expires(self, value: Optional[datetime.datetime]): if value is not None: value = str(google.cloud._helpers._millis_from_datetime(value)) self._properties["expirationTime"] = value @property - def description(self): - """Optional[str]: Description of the model (defaults to - :data:`None`). - """ + def description(self) -> Optional[str]: + """Description of the model (defaults to :data:`None`).""" return self._properties.get("description") @description.setter - def description(self, value): + def description(self, value: Optional[str]): self._properties["description"] = value @property - def friendly_name(self): - """Optional[str]: Title of the table (defaults to :data:`None`). - - Raises: - ValueError: For invalid value types. - """ + def friendly_name(self) -> Optional[str]: + """Title of the table (defaults to :data:`None`).""" return self._properties.get("friendlyName") @friendly_name.setter - def friendly_name(self, value): + def friendly_name(self, value: Optional[str]): self._properties["friendlyName"] = value @property - def labels(self): - """Optional[Dict[str, str]]: Labels for the table. + def labels(self) -> Dict[str, str]: + """Labels for the table. - This method always returns a dict. To change a model's labels, - modify the dict, then call ``Client.update_model``. To delete a - label, set its value to :data:`None` before updating. + This method always returns a dict. To change a model's labels, modify the dict, + then call ``Client.update_model``. To delete a label, set its value to + :data:`None` before updating. """ return self._properties.setdefault("labels", {}) @labels.setter - def labels(self, value): + def labels(self, value: Optional[Dict[str, str]]): if value is None: value = {} self._properties["labels"] = value @property - def encryption_configuration(self): - """Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom - encryption configuration for the model. + def encryption_configuration(self) -> Optional[EncryptionConfiguration]: + """Custom encryption configuration for the model. Custom encryption configuration (e.g., Cloud KMS keys) or :data:`None` if using default encryption. @@ -272,31 +243,30 @@ def encryption_configuration(self): return prop @encryption_configuration.setter - def encryption_configuration(self, value): + def encryption_configuration(self, value: Optional[EncryptionConfiguration]): api_repr = value if value: api_repr = value.to_api_repr() self._properties["encryptionConfiguration"] = api_repr @classmethod - def from_api_repr(cls, resource: dict) -> "Model": + def from_api_repr(cls, resource: Dict[str, Any]) -> "Model": """Factory: construct a model resource given its API representation Args: - resource (Dict[str, object]): + resource: Model resource representation from the API Returns: - google.cloud.bigquery.model.Model: Model parsed from ``resource``. + Model parsed from ``resource``. """ this = cls(None) - # Keep a reference to the resource as a workaround to find unknown - # field values. + + resource = copy.deepcopy(resource) this._properties = resource # Convert from millis-from-epoch to timestamp well-known type. # TODO: Remove this hack once CL 238585470 hits prod. - resource = copy.deepcopy(resource) for training_run in resource.get("trainingRuns", ()): start_time = training_run.get("startTime") if not start_time or "-" in start_time: # Already right format? @@ -304,15 +274,9 @@ def from_api_repr(cls, resource: dict) -> "Model": start_time = datetime_helpers.from_microseconds(1e3 * float(start_time)) training_run["startTime"] = datetime_helpers.to_rfc3339(start_time) - try: - this._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) - except json_format.ParseError: - resource["modelType"] = "MODEL_TYPE_UNSPECIFIED" - this._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) + # NOTE: No check for invalid model types, unlike before when a protobuf model + # did the enum validations - but we don't have that generated enum anymore. + return this def _build_resource(self, filter_fields): @@ -320,18 +284,18 @@ def _build_resource(self, filter_fields): return _helpers._build_resource_from_properties(self, filter_fields) def __repr__(self): - return "Model(reference={})".format(repr(self.reference)) + return f"Model(reference={self.reference!r})" - def to_api_repr(self) -> dict: + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this model. Returns: - Dict[str, object]: Model reference represented as an API resource + Model reference represented as an API resource """ - return json_format.MessageToDict(self._proto) + return copy.deepcopy(self._properties) -class ModelReference(object): +class ModelReference: """ModelReferences are pointers to models. See @@ -339,73 +303,60 @@ class ModelReference(object): """ def __init__(self): - self._proto = types.ModelReference()._pb self._properties = {} @property def project(self): """str: Project bound to the model""" - return self._proto.project_id + return self._properties.get("projectId") @property def dataset_id(self): """str: ID of dataset containing the model.""" - return self._proto.dataset_id + return self._properties.get("datasetId") @property def model_id(self): """str: The model ID.""" - return self._proto.model_id + return self._properties.get("modelId") @property - def path(self): - """str: URL path for the model's APIs.""" - return "/projects/%s/datasets/%s/models/%s" % ( - self._proto.project_id, - self._proto.dataset_id, - self._proto.model_id, - ) + def path(self) -> str: + """URL path for the model's APIs.""" + return f"/projects/{self.project}/datasets/{self.dataset_id}/models/{self.model_id}" @classmethod - def from_api_repr(cls, resource): - """Factory: construct a model reference given its API representation + def from_api_repr(cls, resource: Dict[str, Any]) -> "ModelReference": + """Factory: construct a model reference given its API representation. Args: - resource (Dict[str, object]): + resource: Model reference representation returned from the API Returns: - google.cloud.bigquery.model.ModelReference: - Model reference parsed from ``resource``. + Model reference parsed from ``resource``. """ ref = cls() - # Keep a reference to the resource as a workaround to find unknown - # field values. ref._properties = resource - ref._proto = json_format.ParseDict( - resource, types.ModelReference()._pb, ignore_unknown_fields=True - ) - return ref @classmethod def from_string( - cls, model_id: str, default_project: str = None + cls, model_id: str, default_project: Optional[str] = None ) -> "ModelReference": """Construct a model reference from model ID string. Args: - model_id (str): + model_id: A model ID in standard SQL format. If ``default_project`` is not specified, this must included a project ID, dataset ID, and model ID, each separated by ``.``. - default_project (Optional[str]): + default_project: The project ID to use when ``model_id`` does not include a project ID. Returns: - google.cloud.bigquery.model.ModelReference: - Model reference parsed from ``model_id``. + Model reference parsed from ``model_id``. Raises: ValueError: @@ -419,13 +370,13 @@ def from_string( {"projectId": proj, "datasetId": dset, "modelId": model} ) - def to_api_repr(self) -> dict: + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this model reference. Returns: - Dict[str, object]: Model reference represented as an API resource + Model reference represented as an API resource. """ - return json_format.MessageToDict(self._proto) + return copy.deepcopy(self._properties) def _key(self): """Unique key for this model. @@ -437,7 +388,7 @@ def _key(self): def __eq__(self, other): if not isinstance(other, ModelReference): return NotImplemented - return self._proto == other._proto + return self._properties == other._properties def __ne__(self, other): return not self == other diff --git a/tests/unit/model/test_model.py b/tests/unit/model/test_model.py index 0f264ba00..9b99ca5d5 100644 --- a/tests/unit/model/test_model.py +++ b/tests/unit/model/test_model.py @@ -19,7 +19,6 @@ import pytest import google.cloud._helpers -from google.cloud.bigquery import types KMS_KEY_NAME = "projects/1/locations/us/keyRings/1/cryptoKeys/1" @@ -79,7 +78,7 @@ def test_from_api_repr(target_class): "description": "A friendly description.", "friendlyName": "A friendly name.", "modelType": "LOGISTIC_REGRESSION", - "labels": {"greeting": u"こんにちは"}, + "labels": {"greeting": "こんにちは"}, "trainingRuns": [ { "trainingOptions": {"initialLearnRate": 1.0}, @@ -115,30 +114,24 @@ def test_from_api_repr(target_class): assert got.created == creation_time assert got.modified == modified_time assert got.expires == expiration_time - assert got.description == u"A friendly description." - assert got.friendly_name == u"A friendly name." - assert got.model_type == types.Model.ModelType.LOGISTIC_REGRESSION - assert got.labels == {"greeting": u"こんにちは"} + assert got.description == "A friendly description." + assert got.friendly_name == "A friendly name." + assert got.model_type == "LOGISTIC_REGRESSION" + assert got.labels == {"greeting": "こんにちは"} assert got.encryption_configuration.kms_key_name == KMS_KEY_NAME - assert got.training_runs[0].training_options.initial_learn_rate == 1.0 + assert got.training_runs[0]["trainingOptions"]["initialLearnRate"] == 1.0 assert ( - got.training_runs[0] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[0]["startTime"]) == creation_time ) - assert got.training_runs[1].training_options.initial_learn_rate == 0.5 + assert got.training_runs[1]["trainingOptions"]["initialLearnRate"] == 0.5 assert ( - got.training_runs[1] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[1]["startTime"]) == modified_time ) - assert got.training_runs[2].training_options.initial_learn_rate == 0.25 + assert got.training_runs[2]["trainingOptions"]["initialLearnRate"] == 0.25 assert ( - got.training_runs[2] - .start_time.ToDatetime() - .replace(tzinfo=google.cloud._helpers.UTC) + google.cloud._helpers._rfc3339_to_datetime(got.training_runs[2]["startTime"]) == expiration_time ) @@ -155,14 +148,14 @@ def test_from_api_repr_w_minimal_resource(target_class): } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got.location == "" - assert got.etag == "" + assert got.location is None + assert got.etag is None assert got.created is None assert got.modified is None assert got.expires is None assert got.description is None assert got.friendly_name is None - assert got.model_type == types.Model.ModelType.MODEL_TYPE_UNSPECIFIED + assert got.model_type == "MODEL_TYPE_UNSPECIFIED" assert got.labels == {} assert got.encryption_configuration is None assert len(got.training_runs) == 0 @@ -183,7 +176,7 @@ def test_from_api_repr_w_unknown_fields(target_class): } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got._properties is resource + assert got._properties == resource def test_from_api_repr_w_unknown_type(target_class): @@ -195,12 +188,19 @@ def test_from_api_repr_w_unknown_type(target_class): "datasetId": "my_dataset", "modelId": "my_model", }, - "modelType": "BE_A_GOOD_ROLE_MODEL", + "modelType": "BE_A_GOOD_ROLE_MODEL", # This model type does not exist. } got = target_class.from_api_repr(resource) assert got.reference == ModelReference.from_string("my-project.my_dataset.my_model") - assert got.model_type == 0 - assert got._properties is resource + assert got.model_type == "BE_A_GOOD_ROLE_MODEL" # No checks for invalid types. + assert got._properties == resource + + +def test_from_api_repr_w_missing_reference(target_class): + resource = {} + got = target_class.from_api_repr(resource) + assert got.reference is None + assert got._properties == resource @pytest.mark.parametrize( @@ -338,8 +338,6 @@ def test_repr(target_class): def test_to_api_repr(target_class): - from google.protobuf import json_format - model = target_class("my-proj.my_dset.my_model") resource = { "etag": "abcdefg", @@ -374,8 +372,6 @@ def test_to_api_repr(target_class): "kmsKeyName": "projects/1/locations/us/keyRings/1/cryptoKeys/1" }, } - model._proto = json_format.ParseDict( - resource, types.Model()._pb, ignore_unknown_fields=True - ) + model._properties = resource got = model.to_api_repr() assert got == resource diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index fbe22067d..7abb9fe02 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1829,7 +1829,7 @@ def test_update_model(self): self.assertEqual(updated_model.expires, model.expires) # ETag becomes If-Match header. - model._proto.etag = "etag" + model._properties["etag"] = "etag" client.update_model(model, []) req = conn.api_request.call_args self.assertEqual(req[1]["headers"]["If-Match"], "etag") From c2972813217713feadcda54447a93dd50c5e6719 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 19 Aug 2021 14:28:15 +0200 Subject: [PATCH 05/32] Adjust Routines code to proto removals --- google/cloud/bigquery/routine/routine.py | 32 +++++++----------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index 3956db9c3..f32d72136 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -16,7 +16,7 @@ """Define resources for the BigQuery Routines API.""" -from google.protobuf import json_format +from typing import Optional import google.cloud._helpers from google.cloud.bigquery import _helpers @@ -206,16 +206,12 @@ def return_type(self): if not resource: return resource - output = StandardSqlDataType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlDataType.from_api_repr(resource) @return_type.setter - def return_type(self, value): + def return_type(self, value: StandardSqlDataType): if value: - resource = json_format.MessageToDict(value._pb) + resource = value.to_api_repr() else: resource = None self._properties[self._PROPERTY_TO_API_FIELD["return_type"]] = resource @@ -232,20 +228,14 @@ def return_table_type(self) -> StandardSqlTableType: if not resource: return resource - output = StandardSqlTableType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlTableType.from_api_repr(resource) @return_table_type.setter - def return_table_type(self, value): + def return_table_type(self, value: Optional[StandardSqlTableType]): if not value: resource = None else: - resource = { - "columns": [json_format.MessageToDict(col._pb) for col in value.columns] - } + resource = value.to_api_repr() self._properties[self._PROPERTY_TO_API_FIELD["return_table_type"]] = resource @@ -417,16 +407,12 @@ def data_type(self): if not resource: return resource - output = StandardSqlDataType() - raw_protobuf = json_format.ParseDict( - resource, output._pb, ignore_unknown_fields=True - ) - return type(output).wrap(raw_protobuf) + return StandardSqlDataType.from_api_repr(resource) @data_type.setter def data_type(self, value): if value: - resource = json_format.MessageToDict(value._pb) + resource = value.to_api_repr() else: resource = None self._properties[self._PROPERTY_TO_API_FIELD["data_type"]] = resource From c9acea28b3ac0e5e4f2e5a4f3f5f9d333fdff8b1 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 13 Aug 2021 14:55:19 +0200 Subject: [PATCH 06/32] Adjust imports in samples code --- samples/create_routine.py | 5 ++--- samples/tests/conftest.py | 5 ++--- samples/tests/test_routine_samples.py | 23 +++++++++++------------ 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/samples/create_routine.py b/samples/create_routine.py index 012c7927a..1547fc782 100644 --- a/samples/create_routine.py +++ b/samples/create_routine.py @@ -17,7 +17,6 @@ def create_routine(routine_id): # [START bigquery_create_routine] from google.cloud import bigquery - from google.cloud import bigquery_v2 # Construct a BigQuery client object. client = bigquery.Client() @@ -33,8 +32,8 @@ def create_routine(routine_id): arguments=[ bigquery.RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ) ], diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 0fdacaaec..f84a9a76c 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -20,7 +20,6 @@ import pytest from google.cloud import bigquery -from google.cloud import bigquery_v2 @pytest.fixture(scope="session", autouse=True) @@ -125,8 +124,8 @@ def routine_id(client, dataset_id): routine.arguments = [ bigquery.RoutineArgument( name="x", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ) ] diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index 59ec1fae9..edc799d04 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -13,7 +13,6 @@ # limitations under the License. from google.cloud import bigquery -from google.cloud import bigquery_v2 def test_create_routine(capsys, random_routine_id): @@ -38,22 +37,22 @@ def test_create_routine_ddl(capsys, random_routine_id, client): expected_arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.ARRAY, - array_element_type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.STRUCT, - struct_type=bigquery_v2.types.StandardSqlStructType( + data_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.ARRAY, + array_element_type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.STRUCT, + struct_type=bigquery.types.StandardSqlStructType( fields=[ - bigquery_v2.types.StandardSqlField( + bigquery.types.StandardSqlField( name="name", - type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.STRING + type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.STRING ), ), - bigquery_v2.types.StandardSqlField( + bigquery.types.StandardSqlField( name="val", - type=bigquery_v2.types.StandardSqlDataType( - type_kind=bigquery_v2.types.StandardSqlDataType.TypeKind.INT64 + type=bigquery.types.StandardSqlDataType( + type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 ), ), ] From 24e3d48d7e4ada490a121423a63d718f92444da0 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 19 Aug 2021 14:59:42 +0200 Subject: [PATCH 07/32] Adjust assertion in get routine sample test --- samples/tests/test_routine_samples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index edc799d04..8b27ea1dc 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -82,7 +82,7 @@ def test_get_routine(capsys, routine_id): assert "Type: 'SCALAR_FUNCTION'" in out assert "Language: 'SQL'" in out assert "Name: 'x'" in out - assert "Type: 'type_kind: INT64\n'" in out + assert "type_kind=" in out def test_delete_routine(capsys, routine_id): From 61e3a3401a0f1009ce533ffcb41d0aa1796b91b4 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 20 Aug 2021 16:14:30 +0200 Subject: [PATCH 08/32] Include manual Sql types in docs --- docs/{bigquery_v2 => bigquery}/types.rst | 2 +- docs/conf.py | 1 - docs/reference.rst | 4 +-- owlbot.py | 31 +++++++++--------------- 4 files changed, 14 insertions(+), 24 deletions(-) rename docs/{bigquery_v2 => bigquery}/types.rst (74%) diff --git a/docs/bigquery_v2/types.rst b/docs/bigquery/types.rst similarity index 74% rename from docs/bigquery_v2/types.rst rename to docs/bigquery/types.rst index c36a83e0b..c93879566 100644 --- a/docs/bigquery_v2/types.rst +++ b/docs/bigquery/types.rst @@ -1,7 +1,7 @@ Types for Google Cloud Bigquery v2 API ====================================== -.. automodule:: google.cloud.bigquery_v2.types +.. automodule:: google.cloud.bigquery.types :members: :undoc-members: :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 09f7ea414..a58792ffa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -114,7 +114,6 @@ "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", "samples/snippets/README.rst", - "bigquery_v2/services.rst", # generated by the code generator ] # The reST default role (used for this markup: `text`) to use for all diff --git a/docs/reference.rst b/docs/reference.rst index d8738e67b..505257d24 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -197,9 +197,9 @@ Encryption Configuration Additional Types ================ -Protocol buffer classes for working with the Models API. +Helper SQL type classes. .. toctree:: :maxdepth: 2 - bigquery_v2/types + bigquery/types diff --git a/owlbot.py b/owlbot.py index 09845480a..6543a663c 100644 --- a/owlbot.py +++ b/owlbot.py @@ -63,7 +63,7 @@ s.replace( library / f"google/cloud/bigquery_{library.name}/types/standard_sql.py", r"type_ ", - "type " + "type ", ) s.move( @@ -78,8 +78,8 @@ "noxfile.py", "setup.py", f"scripts/fixup_bigquery_{library.name}_keywords.py", - f"google/cloud/bigquery/__init__.py", - f"google/cloud/bigquery/py.typed", + "google/cloud/bigquery/__init__.py", + "google/cloud/bigquery/py.typed", # There are no public API endpoints for the generated ModelServiceClient, # thus there's no point in generating it and its tests. f"google/cloud/bigquery_{library.name}/services/**", @@ -93,10 +93,7 @@ # Add templated files # ---------------------------------------------------------------------------- templated_files = common.py_library( - cov_level=100, - samples=True, - microgenerator=True, - split_system_tests=True, + cov_level=100, samples=True, microgenerator=True, split_system_tests=True, ) # BigQuery has a custom multiprocessing note @@ -109,7 +106,7 @@ # Include custom SNIPPETS_TESTS job for performance. # https://github.com/googleapis/python-bigquery/issues/191 ".kokoro/presubmit/presubmit.cfg", - ] + ], ) # ---------------------------------------------------------------------------- @@ -121,14 +118,7 @@ s.replace( "docs/conf.py", r'\{"members": True\}', - '{"members": True, "inherited-members": True}' -) - -# Tell Sphinx to ingore autogenerated docs files. -s.replace( - "docs/conf.py", - r'"samples/snippets/README\.rst",', - '\g<0>\n "bigquery_v2/services.rst", # generated by the code generator', + '{"members": True, "inherited-members": True}', ) # ---------------------------------------------------------------------------- @@ -136,13 +126,14 @@ # ---------------------------------------------------------------------------- # Add .pytype to .gitignore -s.replace(".gitignore", r"\.pytest_cache", "\g<0>\n.pytype") +s.replace(".gitignore", r"\.pytest_cache", "\\g<0>\n.pytype") # Add pytype config to setup.cfg s.replace( "setup.cfg", r"universal = 1", - textwrap.dedent(""" \g<0> + textwrap.dedent( + """ \\g<0> [pytype] python_version = 3.8 @@ -150,13 +141,13 @@ google/cloud/ exclude = tests/ - google/cloud/bigquery_v2/ output = .pytype/ disable = # There's some issue with finding some pyi files, thus disabling. # The issue https://github.com/google/pytype/issues/150 is closed, but the # error still occurs for some reason. - pyi-error""") + pyi-error""" + ), ) s.shell.run(["nox", "-s", "blacken"], hide_output=False) From ce755ab8d2c06180f13e72c5aa2faad34448b3f5 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 20 Aug 2021 16:17:55 +0200 Subject: [PATCH 09/32] Remove proto dependencies --- docs/conf.py | 2 -- setup.py | 2 -- testing/constraints-3.6.txt | 2 -- 3 files changed, 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a58792ffa..7ae52bf83 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -363,8 +363,6 @@ "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), - "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), - "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), } diff --git a/setup.py b/setup.py index bac711e4b..de5013c51 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 "google-api-core[grpc] >= 1.29.0, <3.0.0dev", - "proto-plus >= 1.10.0", "google-cloud-bigquery-storage >= 2.0.0, <3.0.0dev", # NOTE: Maintainers, please do not require google-cloud-core>=2.x.x # Until this issue is closed @@ -43,7 +42,6 @@ "google-cloud-core >= 1.4.1, <3.0.0dev", "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3", - "protobuf >= 3.12.0", "pyarrow >= 3.0.0, < 6.0dev", "requests >= 2.18.0, < 3.0.0dev", ] diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index bf1f89f58..17998d199 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -14,8 +14,6 @@ opentelemetry-api==0.11b0 opentelemetry-instrumentation==0.11b0 opentelemetry-sdk==0.11b0 pandas==1.0.0 -proto-plus==1.10.0 -protobuf==3.12.0 pyarrow==3.0.0 requests==2.18.0 six==1.13.0 From 199725fc29686a4c09deb32af716a368a9f88a8d Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 20 Aug 2021 16:42:32 +0200 Subject: [PATCH 10/32] Omit the generated bigquery_v2 code altogether --- owlbot.py | 48 +++--------------------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/owlbot.py b/owlbot.py index 6543a663c..a0bd1aab7 100644 --- a/owlbot.py +++ b/owlbot.py @@ -24,48 +24,6 @@ default_version = "v2" for library in s.get_staging_dirs(default_version): - # Do not expose ModelServiceClient and ModelServiceAsyncClient, as there - # is no public API endpoint for the models service. - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"from \.services\.model_service import ModelServiceClient", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"from \.services\.model_service import ModelServiceAsyncClient", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"""["']ModelServiceClient["'],""", - "", - ) - - s.replace( - library / f"google/cloud/bigquery_{library.name}/__init__.py", - r"""["']ModelServiceAsyncClient["'],""", - "", - ) - - # Adjust Model docstring so that Sphinx does not think that "predicted_" is - # a reference to something, issuing a false warning. - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/model.py", - r'will have a "predicted_"', - "will have a `predicted_`", - ) - - # Avoid breaking change due to change in field renames. - # https://github.com/googleapis/python-bigquery/issues/319 - s.replace( - library / f"google/cloud/bigquery_{library.name}/types/standard_sql.py", - r"type_ ", - "type ", - ) - s.move( library, excludes=[ @@ -80,9 +38,9 @@ f"scripts/fixup_bigquery_{library.name}_keywords.py", "google/cloud/bigquery/__init__.py", "google/cloud/bigquery/py.typed", - # There are no public API endpoints for the generated ModelServiceClient, - # thus there's no point in generating it and its tests. - f"google/cloud/bigquery_{library.name}/services/**", + # The library does not use any protos (it implements its own type classes), + # thus omit the generated code altogether. + f"google/cloud/bigquery_{library.name}/**", f"tests/unit/gapic/bigquery_{library.name}/**", ], ) From 6c167a718076545844ac1bedf657156ab3ac7c2d Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 31 Aug 2021 18:55:40 +0200 Subject: [PATCH 11/32] Move SQL types a level up, expose at top level --- google/cloud/bigquery/__init__.py | 9 +++ google/cloud/bigquery/enums.py | 6 +- google/cloud/bigquery/model.py | 6 +- google/cloud/bigquery/routine/routine.py | 8 +-- google/cloud/bigquery/schema.py | 64 ++++++++++--------- .../bigquery/{types => }/standard_sql.py | 0 google/cloud/bigquery/types/__init__.py | 30 --------- samples/create_routine.py | 4 +- samples/tests/conftest.py | 4 +- samples/tests/test_routine_samples.py | 22 +++---- tests/system/test_client.py | 14 ++-- .../enums/test_standard_sql_data_types.py | 2 +- tests/unit/routine/test_routine.py | 28 ++++---- tests/unit/routine/test_routine_argument.py | 12 ++-- tests/unit/test_client.py | 4 +- tests/unit/test_schema.py | 36 +++++------ tests/unit/test_standard_sql_types.py | 30 ++++----- 17 files changed, 130 insertions(+), 149 deletions(-) rename google/cloud/bigquery/{types => }/standard_sql.py (100%) delete mode 100644 google/cloud/bigquery/types/__init__.py diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 5529f9b2e..1bd2adc53 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -87,6 +87,10 @@ from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.routine import RoutineType from google.cloud.bigquery.schema import SchemaField +from google.cloud.bigquery.standard_sql import StandardSqlDataType +from google.cloud.bigquery.standard_sql import StandardSqlField +from google.cloud.bigquery.standard_sql import StandardSqlStructType +from google.cloud.bigquery.standard_sql import StandardSqlTableType from google.cloud.bigquery.table import PartitionRange from google.cloud.bigquery.table import RangePartitioning from google.cloud.bigquery.table import Row @@ -151,6 +155,11 @@ "ScriptOptions", "TransactionInfo", "DEFAULT_RETRY", + # Standard SQL types + "StandardSqlDataType", + "StandardSqlField", + "StandardSqlStructType", + "StandardSqlTableType", # Enum Constants "enums", "AutoRowIDs", diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 5f001d8c4..239c1f34d 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -14,7 +14,7 @@ import enum -from google.cloud.bigquery import types +from google.cloud.bigquery import standard_sql from google.cloud.bigquery.query import ScalarQueryParameterType @@ -206,11 +206,11 @@ def _make_sql_scalars_enum(): "StandardSqlDataTypes", ( (member.name, member.value) - for member in types.StandardSqlDataType.TypeKind + for member in standard_sql.StandardSqlDataType.TypeKind if member.name in _SQL_SCALAR_TYPES ), ) - new_enum.__doc__ = types.StandardSqlDataType.__doc__ + new_enum.__doc__ = standard_sql.StandardSqlDataType.__doc__ return new_enum diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 21c2417fe..f1315dce9 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -23,7 +23,7 @@ import google.cloud._helpers from google.api_core import datetime_helpers from google.cloud.bigquery import _helpers -from google.cloud.bigquery import types +from google.cloud.bigquery import standard_sql from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration @@ -155,7 +155,7 @@ def training_runs(self) -> Sequence[Dict[str, Any]]: return self._properties.get("trainingRuns", []) @property - def feature_columns(self) -> Sequence[types.StandardSqlField]: + def feature_columns(self) -> Sequence[standard_sql.StandardSqlField]: """Input feature columns that were used to train this model. Read-only. @@ -163,7 +163,7 @@ def feature_columns(self) -> Sequence[types.StandardSqlField]: return self._properties.get("featureColumns", []) @property - def label_columns(self) -> Sequence[types.StandardSqlField]: + def label_columns(self) -> Sequence[standard_sql.StandardSqlField]: """Label columns that were used to train this model. The output of the model will have a ``predicted_`` prefix to these columns. diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index f32d72136..677fb1178 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -20,8 +20,8 @@ import google.cloud._helpers from google.cloud.bigquery import _helpers -from google.cloud.bigquery.types import StandardSqlDataType -from google.cloud.bigquery.types import StandardSqlTableType +from google.cloud.bigquery.standard_sql import StandardSqlDataType +from google.cloud.bigquery.standard_sql import StandardSqlTableType class RoutineType: @@ -190,7 +190,7 @@ def arguments(self, value): @property def return_type(self): - """google.cloud.bigquery.types.StandardSqlDataType: Return type of + """google.cloud.bigquery.StandardSqlDataType: Return type of the routine. If absent, the return type is inferred from @@ -397,7 +397,7 @@ def mode(self, value): @property def data_type(self): - """Optional[google.cloud.bigquery.types.StandardSqlDataType]: Type + """Optional[google.cloud.bigquery.StandardSqlDataType]: Type of a variable, e.g., a function argument. See: diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 7ae3a7d05..cb58ed93f 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -17,7 +17,7 @@ import collections from typing import Optional -from google.cloud.bigquery import types +from google.cloud.bigquery import standard_sql _DEFAULT_VALUE = object() @@ -27,26 +27,26 @@ # https://cloud.google.com/bigquery/data-types#legacy_sql_data_types # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types LEGACY_TO_STANDARD_TYPES = { - "STRING": types.StandardSqlDataType.TypeKind.STRING, - "BYTES": types.StandardSqlDataType.TypeKind.BYTES, - "INTEGER": types.StandardSqlDataType.TypeKind.INT64, - "INT64": types.StandardSqlDataType.TypeKind.INT64, - "FLOAT": types.StandardSqlDataType.TypeKind.FLOAT64, - "FLOAT64": types.StandardSqlDataType.TypeKind.FLOAT64, - "NUMERIC": types.StandardSqlDataType.TypeKind.NUMERIC, - "BIGNUMERIC": types.StandardSqlDataType.TypeKind.BIGNUMERIC, - "BOOLEAN": types.StandardSqlDataType.TypeKind.BOOL, - "BOOL": types.StandardSqlDataType.TypeKind.BOOL, - "GEOGRAPHY": types.StandardSqlDataType.TypeKind.GEOGRAPHY, - "RECORD": types.StandardSqlDataType.TypeKind.STRUCT, - "STRUCT": types.StandardSqlDataType.TypeKind.STRUCT, - "TIMESTAMP": types.StandardSqlDataType.TypeKind.TIMESTAMP, - "DATE": types.StandardSqlDataType.TypeKind.DATE, - "TIME": types.StandardSqlDataType.TypeKind.TIME, - "DATETIME": types.StandardSqlDataType.TypeKind.DATETIME, + "STRING": standard_sql.StandardSqlDataType.TypeKind.STRING, + "BYTES": standard_sql.StandardSqlDataType.TypeKind.BYTES, + "INTEGER": standard_sql.StandardSqlDataType.TypeKind.INT64, + "INT64": standard_sql.StandardSqlDataType.TypeKind.INT64, + "FLOAT": standard_sql.StandardSqlDataType.TypeKind.FLOAT64, + "FLOAT64": standard_sql.StandardSqlDataType.TypeKind.FLOAT64, + "NUMERIC": standard_sql.StandardSqlDataType.TypeKind.NUMERIC, + "BIGNUMERIC": standard_sql.StandardSqlDataType.TypeKind.BIGNUMERIC, + "BOOLEAN": standard_sql.StandardSqlDataType.TypeKind.BOOL, + "BOOL": standard_sql.StandardSqlDataType.TypeKind.BOOL, + "GEOGRAPHY": standard_sql.StandardSqlDataType.TypeKind.GEOGRAPHY, + "RECORD": standard_sql.StandardSqlDataType.TypeKind.STRUCT, + "STRUCT": standard_sql.StandardSqlDataType.TypeKind.STRUCT, + "TIMESTAMP": standard_sql.StandardSqlDataType.TypeKind.TIMESTAMP, + "DATE": standard_sql.StandardSqlDataType.TypeKind.DATE, + "TIME": standard_sql.StandardSqlDataType.TypeKind.TIME, + "DATETIME": standard_sql.StandardSqlDataType.TypeKind.DATETIME, # no direct conversion from ARRAY, the latter is represented by mode="REPEATED" } -"""String names of the legacy SQL types to integer codes of Standard SQL types.""" +"""String names of the legacy SQL types to integer codes of Standard SQL standard_sql.""" class SchemaField(object): @@ -285,24 +285,26 @@ def _key(self): policy_tags, ) - def to_standard_sql(self) -> types.StandardSqlField: + def to_standard_sql(self) -> standard_sql.StandardSqlField: """Return the field as the standard SQL field representation object.""" - sql_type = types.StandardSqlDataType() + sql_type = standard_sql.StandardSqlDataType() if self.mode == "REPEATED": - sql_type.type_kind = types.StandardSqlDataType.TypeKind.ARRAY + sql_type.type_kind = standard_sql.StandardSqlDataType.TypeKind.ARRAY else: sql_type.type_kind = LEGACY_TO_STANDARD_TYPES.get( self.field_type, - types.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + standard_sql.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, ) - if sql_type.type_kind == types.StandardSqlDataType.TypeKind.ARRAY: # noqa: E721 + if ( + sql_type.type_kind == standard_sql.StandardSqlDataType.TypeKind.ARRAY + ): # noqa: E721 array_element_type = LEGACY_TO_STANDARD_TYPES.get( self.field_type, - types.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + standard_sql.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, ) - sql_type.array_element_type = types.StandardSqlDataType( + sql_type.array_element_type = standard_sql.StandardSqlDataType( type_kind=array_element_type ) @@ -310,20 +312,20 @@ def to_standard_sql(self) -> types.StandardSqlField: # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type if ( array_element_type - == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 + == standard_sql.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 ): - sql_type.array_element_type.struct_type = types.StandardSqlStructType( + sql_type.array_element_type.struct_type = standard_sql.StandardSqlStructType( fields=(field.to_standard_sql() for field in self.fields) ) elif ( sql_type.type_kind - == types.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 + == standard_sql.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 ): - sql_type.struct_type = types.StandardSqlStructType( + sql_type.struct_type = standard_sql.StandardSqlStructType( fields=(field.to_standard_sql() for field in self.fields) ) - return types.StandardSqlField(name=self.name, type=sql_type) + return standard_sql.StandardSqlField(name=self.name, type=sql_type) def __eq__(self, other): if not isinstance(other, SchemaField): diff --git a/google/cloud/bigquery/types/standard_sql.py b/google/cloud/bigquery/standard_sql.py similarity index 100% rename from google/cloud/bigquery/types/standard_sql.py rename to google/cloud/bigquery/standard_sql.py diff --git a/google/cloud/bigquery/types/__init__.py b/google/cloud/bigquery/types/__init__.py deleted file mode 100644 index 43f3e40bf..000000000 --- a/google/cloud/bigquery/types/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: This subpackage define type classes that were previously generated from protobuf -# definitions in library versions before 3.0. - -from .standard_sql import ( - StandardSqlDataType, - StandardSqlField, - StandardSqlStructType, - StandardSqlTableType, -) - -__all__ = ( - "StandardSqlDataType", - "StandardSqlField", - "StandardSqlStructType", - "StandardSqlTableType", -) diff --git a/samples/create_routine.py b/samples/create_routine.py index 1547fc782..3845548ed 100644 --- a/samples/create_routine.py +++ b/samples/create_routine.py @@ -32,8 +32,8 @@ def create_routine(routine_id): arguments=[ bigquery.RoutineArgument( name="x", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 ), ) ], diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index f84a9a76c..5d4a9f49f 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -124,8 +124,8 @@ def routine_id(client, dataset_id): routine.arguments = [ bigquery.RoutineArgument( name="x", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 ), ) ] diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index 8b27ea1dc..490e914b0 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -37,22 +37,22 @@ def test_create_routine_ddl(capsys, random_routine_id, client): expected_arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.ARRAY, - array_element_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.STRUCT, - struct_type=bigquery.types.StandardSqlStructType( + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.ARRAY, + array_element_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.STRUCT, + struct_type=bigquery.StandardSqlStructType( fields=[ - bigquery.types.StandardSqlField( + bigquery.StandardSqlField( name="name", - type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.STRING + type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.STRING ), ), - bigquery.types.StandardSqlField( + bigquery.StandardSqlField( name="val", - type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 ), ), ] diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 894b28d58..de5e9be1e 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -2180,8 +2180,8 @@ def test_insert_rows_nested_nested_dictionary(self): def test_create_routine(self): routine_name = "test_routine" dataset = self.temp_dataset(_make_dataset_id("create_routine")) - float64_type = bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.FLOAT64 + float64_type = bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.FLOAT64 ) routine = bigquery.Routine( dataset.routine(routine_name), @@ -2195,8 +2195,8 @@ def test_create_routine(self): routine.arguments = [ bigquery.RoutineArgument( name="arr", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.ARRAY, + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlDataType.TypeKind.ARRAY, array_element_type=float64_type, ), ) @@ -2217,9 +2217,9 @@ def test_create_routine(self): def test_create_tvf_routine(self): from google.cloud.bigquery import Routine, RoutineArgument, RoutineType - StandardSqlDataType = bigquery.types.StandardSqlDataType - StandardSqlField = bigquery.types.StandardSqlField - StandardSqlTableType = bigquery.types.StandardSqlTableType + StandardSqlDataType = bigquery.StandardSqlDataType + StandardSqlField = bigquery.StandardSqlField + StandardSqlTableType = bigquery.StandardSqlTableType INT64 = StandardSqlDataType.TypeKind.INT64 STRING = StandardSqlDataType.TypeKind.STRING diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py index d3c08a099..d26528567 100644 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ b/tests/unit/enums/test_standard_sql_data_types.py @@ -32,7 +32,7 @@ def enum_under_test(): @pytest.fixture def gapic_enum(): """The referential autogenerated enum the enum under test is based on.""" - from google.cloud.bigquery.types import StandardSqlDataType + from google.cloud.bigquery.standard_sql import StandardSqlDataType return StandardSqlDataType.TypeKind diff --git a/tests/unit/routine/test_routine.py b/tests/unit/routine/test_routine.py index 66b05d2fb..c5701d185 100644 --- a/tests/unit/routine/test_routine.py +++ b/tests/unit/routine/test_routine.py @@ -61,15 +61,15 @@ def test_ctor_w_properties(target_class): arguments = [ RoutineArgument( name="x", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ), ) ] body = "x * 3" language = "SQL" - return_type = bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + return_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ) type_ = "SCALAR_FUNCTION" description = "A routine description." @@ -145,15 +145,15 @@ def test_from_api_repr(target_class): assert actual_routine.arguments == [ RoutineArgument( name="x", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ), ) ] assert actual_routine.body == "42" assert actual_routine.language == "SQL" - assert actual_routine.return_type == bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + assert actual_routine.return_type == bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ) assert actual_routine.return_table_type is None assert actual_routine.type_ == "SCALAR_FUNCTION" @@ -167,9 +167,9 @@ def test_from_api_repr_tvf_function(target_class): from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.routine import RoutineType - StandardSqlDataType = bigquery.types.StandardSqlDataType - StandardSqlField = bigquery.types.StandardSqlField - StandardSqlTableType = bigquery.types.StandardSqlTableType + StandardSqlDataType = bigquery.standard_sql.StandardSqlDataType + StandardSqlField = bigquery.standard_sql.StandardSqlField + StandardSqlTableType = bigquery.standard_sql.StandardSqlTableType creation_time = datetime.datetime( 2010, 5, 19, 16, 0, 0, tzinfo=google.cloud._helpers.UTC @@ -459,9 +459,9 @@ def test_set_return_table_type_w_none(object_under_test): def test_set_return_table_type_w_not_none(object_under_test): - StandardSqlDataType = bigquery.types.StandardSqlDataType - StandardSqlField = bigquery.types.StandardSqlField - StandardSqlTableType = bigquery.types.StandardSqlTableType + StandardSqlDataType = bigquery.standard_sql.StandardSqlDataType + StandardSqlField = bigquery.standard_sql.StandardSqlField + StandardSqlTableType = bigquery.standard_sql.StandardSqlTableType table_type = StandardSqlTableType( columns=[ diff --git a/tests/unit/routine/test_routine_argument.py b/tests/unit/routine/test_routine_argument.py index d30830444..0f1701f95 100644 --- a/tests/unit/routine/test_routine_argument.py +++ b/tests/unit/routine/test_routine_argument.py @@ -27,8 +27,8 @@ def target_class(): def test_ctor(target_class): - data_type = bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ) actual_arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type @@ -50,8 +50,8 @@ def test_from_api_repr(target_class): assert actual_arg.name == "field_name" assert actual_arg.kind == "FIXED_TYPE" assert actual_arg.mode == "IN" - assert actual_arg.data_type == bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + assert actual_arg.data_type == bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ) @@ -71,8 +71,8 @@ def test_from_api_repr_w_unknown_fields(target_class): def test_eq(target_class): - data_type = bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type = bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ) arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 2bedffbe7..d491ead3f 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1859,8 +1859,8 @@ def test_update_routine(self): routine.arguments = [ RoutineArgument( name="x", - data_type=bigquery.types.StandardSqlDataType( - type_kind=bigquery.types.StandardSqlDataType.TypeKind.INT64 + data_type=bigquery.standard_sql.StandardSqlDataType( + type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 ), ) ] diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 8ff70b595..a05ca7178 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud.bigquery.types.standard_sql import StandardSqlStructType +from google.cloud.bigquery.standard_sql import StandardSqlStructType from google.cloud.bigquery.schema import PolicyTagList import unittest @@ -29,9 +29,9 @@ def _get_target_class(): @staticmethod def _get_standard_sql_data_type_class(): - from google.cloud.bigquery import types + from google.cloud.bigquery import standard_sql - return types.StandardSqlDataType + return standard_sql.StandardSqlDataType def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) @@ -225,7 +225,7 @@ def test_to_standard_sql_simple_type(self): self.assertEqual(standard_field.type.type_kind, standard_type) def test_to_standard_sql_struct_type(self): - from google.cloud.bigquery import types + from google.cloud.bigquery import standard_sql # Expected result object: # @@ -259,33 +259,33 @@ def test_to_standard_sql_struct_type(self): sql_type = self._get_standard_sql_data_type_class() # level 2 fields - sub_sub_field_date = types.StandardSqlField( + sub_sub_field_date = standard_sql.StandardSqlField( name="date_field", type=sql_type(type_kind=sql_type.TypeKind.DATE) ) - sub_sub_field_time = types.StandardSqlField( + sub_sub_field_time = standard_sql.StandardSqlField( name="time_field", type=sql_type(type_kind=sql_type.TypeKind.TIME) ) # level 1 fields - sub_field_struct = types.StandardSqlField( + sub_field_struct = standard_sql.StandardSqlField( name="last_used", type=sql_type( type_kind=sql_type.TypeKind.STRUCT, - struct_type=types.StandardSqlStructType( + struct_type=standard_sql.StandardSqlStructType( fields=[sub_sub_field_date, sub_sub_field_time] ), ), ) - sub_field_bytes = types.StandardSqlField( + sub_field_bytes = standard_sql.StandardSqlField( name="image_content", type=sql_type(type_kind=sql_type.TypeKind.BYTES) ) # level 0 (top level) - expected_result = types.StandardSqlField( + expected_result = standard_sql.StandardSqlField( name="image_usage", type=sql_type( type_kind=sql_type.TypeKind.STRUCT, - struct_type=types.StandardSqlStructType( + struct_type=standard_sql.StandardSqlStructType( fields=[sub_field_bytes, sub_field_struct] ), ), @@ -307,7 +307,7 @@ def test_to_standard_sql_struct_type(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_simple(self): - from google.cloud.bigquery import types + from google.cloud.bigquery import standard_sql sql_type = self._get_standard_sql_data_type_class() @@ -316,7 +316,7 @@ def test_to_standard_sql_array_type_simple(self): type_kind=sql_type.TypeKind.ARRAY, array_element_type=sql_type(type_kind=sql_type.TypeKind.INT64), ) - expected_result = types.StandardSqlField( + expected_result = standard_sql.StandardSqlField( name="valid_numbers", type=expected_sql_type ) @@ -327,18 +327,18 @@ def test_to_standard_sql_array_type_simple(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_array_type_struct(self): - from google.cloud.bigquery import types + from google.cloud.bigquery import standard_sql sql_type = self._get_standard_sql_data_type_class() # define person STRUCT - name_field = types.StandardSqlField( + name_field = standard_sql.StandardSqlField( name="name", type=sql_type(type_kind=sql_type.TypeKind.STRING) ) - age_field = types.StandardSqlField( + age_field = standard_sql.StandardSqlField( name="age", type=sql_type(type_kind=sql_type.TypeKind.INT64) ) - person_struct = types.StandardSqlField( + person_struct = standard_sql.StandardSqlField( name="person_info", type=sql_type( type_kind=sql_type.TypeKind.STRUCT, @@ -350,7 +350,7 @@ def test_to_standard_sql_array_type_struct(self): expected_sql_type = sql_type( type_kind=sql_type.TypeKind.ARRAY, array_element_type=person_struct.type ) - expected_result = types.StandardSqlField( + expected_result = standard_sql.StandardSqlField( name="known_people", type=expected_sql_type ) diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index ea65800eb..5e52e8ecb 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -18,7 +18,7 @@ class TestStandardSqlDataType: @staticmethod def _get_target_class(): - from google.cloud.bigquery.types.standard_sql import StandardSqlDataType + from google.cloud.bigquery.standard_sql import StandardSqlDataType return StandardSqlDataType @@ -26,8 +26,8 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_ctor_w_both_array_element_and_struct_types(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField - from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType StandardSqlDataType = self._get_target_class() TypeKind = StandardSqlDataType.TypeKind @@ -97,8 +97,8 @@ def test_to_api_repr_struct_type_field_types_missing(self): assert result == {"typeKind": "STRUCT", "structType": None} def test_to_api_repr_struct_type_w_field_types(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField - from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType StandardSqlDataType = self._get_target_class() TypeKind = StandardSqlDataType.TypeKind @@ -201,8 +201,8 @@ def test_from_api_repr_array_type_missing_element_type(self): assert result == expected def test_from_api_repr_struct_type_nested(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField - from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() TypeKind = klass.TypeKind @@ -255,7 +255,7 @@ def test_from_api_repr_struct_type_nested(self): assert result == expected def test_from_api_repr_struct_type_missing_struct_info(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() resource = {"typeKind": "STRUCT"} @@ -270,8 +270,8 @@ def test_from_api_repr_struct_type_missing_struct_info(self): assert result == expected def test_from_api_repr_struct_type_incomplete_field_info(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField - from google.cloud.bigquery.types.standard_sql import StandardSqlStructType + from google.cloud.bigquery.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() TypeKind = klass.TypeKind @@ -303,7 +303,7 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): class TestStandardSqlTableType: @staticmethod def _get_target_class(): - from google.cloud.bigquery.types.standard_sql import StandardSqlTableType + from google.cloud.bigquery.standard_sql import StandardSqlTableType return StandardSqlTableType @@ -311,7 +311,7 @@ def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) def test_columns_shallow_copy(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlField columns = [ StandardSqlField("foo"), @@ -331,7 +331,7 @@ def test_to_api_repr_no_columns(self): assert result == {"columns": []} def test_to_api_repr_with_columns(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlField columns = [StandardSqlField("foo"), StandardSqlField("bar")] instance = self._make_one(columns=columns) @@ -349,8 +349,8 @@ def test_from_api_repr_missing_columns(self): assert result.columns == [] def test_from_api_repr_with_incomplete_columns(self): - from google.cloud.bigquery.types.standard_sql import StandardSqlDataType - from google.cloud.bigquery.types.standard_sql import StandardSqlField + from google.cloud.bigquery.standard_sql import StandardSqlDataType + from google.cloud.bigquery.standard_sql import StandardSqlField resource = { "columns": [ From 2b1a9158898358b7b150c9044761494590c4a1d7 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 31 Aug 2021 18:56:53 +0200 Subject: [PATCH 12/32] Remove a comment that is not that informative --- google/cloud/bigquery/model.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index f1315dce9..d5a31d6af 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -274,9 +274,6 @@ def from_api_repr(cls, resource: Dict[str, Any]) -> "Model": start_time = datetime_helpers.from_microseconds(1e3 * float(start_time)) training_run["startTime"] = datetime_helpers.to_rfc3339(start_time) - # NOTE: No check for invalid model types, unlike before when a protobuf model - # did the enum validations - but we don't have that generated enum anymore. - return this def _build_resource(self, filter_fields): From c73e2010ec22ef9dc9605447cb51428950740f80 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 31 Aug 2021 19:03:03 +0200 Subject: [PATCH 13/32] Adjust docs paths to moved SQL types --- docs/bigquery/{types.rst => standard_sql.rst} | 2 +- docs/reference.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename docs/bigquery/{types.rst => standard_sql.rst} (72%) diff --git a/docs/bigquery/types.rst b/docs/bigquery/standard_sql.rst similarity index 72% rename from docs/bigquery/types.rst rename to docs/bigquery/standard_sql.rst index c93879566..bd52bb78f 100644 --- a/docs/bigquery/types.rst +++ b/docs/bigquery/standard_sql.rst @@ -1,7 +1,7 @@ Types for Google Cloud Bigquery v2 API ====================================== -.. automodule:: google.cloud.bigquery.types +.. automodule:: google.cloud.bigquery.standard_sql :members: :undoc-members: :show-inheritance: diff --git a/docs/reference.rst b/docs/reference.rst index 505257d24..128dee718 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -202,4 +202,4 @@ Helper SQL type classes. .. toctree:: :maxdepth: 2 - bigquery/types + bigquery/standard_sql From d465ce68e090c2e3b2b6bab6623a3e194bbb89ba Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 31 Aug 2021 19:25:02 +0200 Subject: [PATCH 14/32] Fix invalid Sphinx directives --- google/cloud/bigquery/job/copy_.py | 2 +- google/cloud/bigquery/job/extract.py | 2 +- google/cloud/bigquery/job/load.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/job/copy_.py b/google/cloud/bigquery/job/copy_.py index c6ee98944..f0dd3d668 100644 --- a/google/cloud/bigquery/job/copy_.py +++ b/google/cloud/bigquery/job/copy_.py @@ -240,7 +240,7 @@ def to_api_repr(self): def from_api_repr(cls, resource, client): """Factory: construct a job given its API representation - .. note: + .. note:: This method assumes that the project found in the resource matches the client's project. diff --git a/google/cloud/bigquery/job/extract.py b/google/cloud/bigquery/job/extract.py index 3373bcdef..52aa036c9 100644 --- a/google/cloud/bigquery/job/extract.py +++ b/google/cloud/bigquery/job/extract.py @@ -244,7 +244,7 @@ def to_api_repr(self): def from_api_repr(cls, resource: dict, client) -> "ExtractJob": """Factory: construct a job given its API representation - .. note: + .. note:: This method assumes that the project found in the resource matches the client's project. diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index aee055c1c..b12c3e621 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -800,7 +800,7 @@ def to_api_repr(self): def from_api_repr(cls, resource: dict, client) -> "LoadJob": """Factory: construct a job given its API representation - .. note: + .. note:: This method assumes that the project found in the resource matches the client's project. From fa4d67082da61a130d4842892e349e9d9fd79ba2 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 6 Sep 2021 19:11:44 +0200 Subject: [PATCH 15/32] Rename TypeKind enum to StandardSqlTypeNames --- google/cloud/bigquery/__init__.py | 4 + google/cloud/bigquery/dbapi/_helpers.py | 6 +- google/cloud/bigquery/enums.py | 52 +++++----- google/cloud/bigquery/query.py | 24 ++++- google/cloud/bigquery/schema.py | 57 +++++------ google/cloud/bigquery/standard_sql.py | 45 +++------ samples/create_routine.py | 2 +- samples/tests/conftest.py | 2 +- samples/tests/test_routine_samples.py | 10 +- tests/system/test_client.py | 15 ++- .../enums/test_standard_sql_data_types.py | 22 ++--- tests/unit/routine/test_routine.py | 20 ++-- tests/unit/routine/test_routine_argument.py | 6 +- tests/unit/test_client.py | 2 +- tests/unit/test_dbapi__helpers.py | 6 +- tests/unit/test_query.py | 4 +- tests/unit/test_schema.py | 48 +++++----- tests/unit/test_standard_sql_types.py | 94 ++++++++++--------- 18 files changed, 218 insertions(+), 201 deletions(-) diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 1bd2adc53..0b6968c65 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -42,6 +42,7 @@ from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.enums import SqlTypeNames from google.cloud.bigquery.enums import StandardSqlDataTypes +from google.cloud.bigquery.enums import StandardSqlTypeNames from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.external_config import BigtableOptions from google.cloud.bigquery.external_config import BigtableColumnFamily @@ -77,6 +78,7 @@ from google.cloud.bigquery.query import ArrayQueryParameterType from google.cloud.bigquery.query import ScalarQueryParameter from google.cloud.bigquery.query import ScalarQueryParameterType +from google.cloud.bigquery.query import SqlParameterScalarTypes from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import StructQueryParameterType from google.cloud.bigquery.query import UDFResource @@ -112,6 +114,7 @@ "StructQueryParameter", "ArrayQueryParameterType", "ScalarQueryParameterType", + "SqlParameterScalarTypes", "StructQueryParameterType", # Datasets "Dataset", @@ -178,6 +181,7 @@ "SourceFormat", "SqlTypeNames", "StandardSqlDataTypes", + "StandardSqlTypeNames", "WriteDisposition", # EncryptionConfiguration "EncryptionConfiguration", diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py index 9c134b47c..8ca9bad05 100644 --- a/google/cloud/bigquery/dbapi/_helpers.py +++ b/google/cloud/bigquery/dbapi/_helpers.py @@ -22,7 +22,7 @@ import typing from google.cloud import bigquery -from google.cloud.bigquery import table, enums, query +from google.cloud.bigquery import table, query from google.cloud.bigquery.dbapi import exceptions @@ -48,7 +48,7 @@ def _parameter_type(name, value, query_parameter_type=None, value_doc=""): query_parameter_type = type_parameters_re.sub("", query_parameter_type) try: parameter_type = getattr( - enums.SqlParameterScalarTypes, query_parameter_type.upper() + query.SqlParameterScalarTypes, query_parameter_type.upper() )._type except AttributeError: raise exceptions.ProgrammingError( @@ -185,7 +185,7 @@ def _parse_type( # Strip type parameters type_ = type_parameters_re.sub("", type_).strip() try: - type_ = getattr(enums.SqlParameterScalarTypes, type_.upper()) + type_ = getattr(query.SqlParameterScalarTypes, type_.upper()) except AttributeError: raise exceptions.ProgrammingError( f"The given parameter type, {type_}," diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 239c1f34d..ff803c819 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -14,9 +14,6 @@ import enum -from google.cloud.bigquery import standard_sql -from google.cloud.bigquery.query import ScalarQueryParameterType - class AutoRowIDs(enum.Enum): """How to handle automatic insert IDs when inserting rows as a stream.""" @@ -177,6 +174,29 @@ class KeyResultStatementKind: FIRST_SELECT = "FIRST_SELECT" +class StandardSqlTypeNames(str, enum.Enum): + def _generate_next_value_(name, start, count, last_values): + return name + + TYPE_KIND_UNSPECIFIED = enum.auto() + INT64 = enum.auto() + BOOL = enum.auto() + FLOAT64 = enum.auto() + STRING = enum.auto() + BYTES = enum.auto() + TIMESTAMP = enum.auto() + DATE = enum.auto() + TIME = enum.auto() + DATETIME = enum.auto() + INTERVAL = enum.auto() + GEOGRAPHY = enum.auto() + NUMERIC = enum.auto() + BIGNUMERIC = enum.auto() + JSON = enum.auto() + ARRAY = enum.auto() + STRUCT = enum.auto() + + _SQL_SCALAR_TYPES = frozenset( ( "INT64", @@ -206,11 +226,11 @@ def _make_sql_scalars_enum(): "StandardSqlDataTypes", ( (member.name, member.value) - for member in standard_sql.StandardSqlDataType.TypeKind + for member in StandardSqlTypeNames if member.name in _SQL_SCALAR_TYPES ), ) - new_enum.__doc__ = standard_sql.StandardSqlDataType.__doc__ + new_enum.__doc__ = "Scalar standard SQL types." return new_enum @@ -242,28 +262,6 @@ class SqlTypeNames(str, enum.Enum): DATETIME = "DATETIME" -class SqlParameterScalarTypes: - """Supported scalar SQL query parameter types as type objects.""" - - BOOL = ScalarQueryParameterType("BOOL") - BOOLEAN = ScalarQueryParameterType("BOOL") - BIGDECIMAL = ScalarQueryParameterType("BIGNUMERIC") - BIGNUMERIC = ScalarQueryParameterType("BIGNUMERIC") - BYTES = ScalarQueryParameterType("BYTES") - DATE = ScalarQueryParameterType("DATE") - DATETIME = ScalarQueryParameterType("DATETIME") - DECIMAL = ScalarQueryParameterType("NUMERIC") - FLOAT = ScalarQueryParameterType("FLOAT64") - FLOAT64 = ScalarQueryParameterType("FLOAT64") - GEOGRAPHY = ScalarQueryParameterType("GEOGRAPHY") - INT64 = ScalarQueryParameterType("INT64") - INTEGER = ScalarQueryParameterType("INT64") - NUMERIC = ScalarQueryParameterType("NUMERIC") - STRING = ScalarQueryParameterType("STRING") - TIME = ScalarQueryParameterType("TIME") - TIMESTAMP = ScalarQueryParameterType("TIMESTAMP") - - class WriteDisposition(object): """Specifies the action that occurs if destination table already exists. diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 1f449f189..d58d46fd9 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -339,7 +339,7 @@ class ScalarQueryParameter(_AbstractQueryParameter): type_: Name of parameter type. See :class:`google.cloud.bigquery.enums.SqlTypeNames` and - :class:`google.cloud.bigquery.enums.SqlParameterScalarTypes` for + :class:`google.cloud.bigquery.query.SqlParameterScalarTypes` for supported types. value: @@ -750,6 +750,28 @@ def __repr__(self): return "StructQueryParameter{}".format(self._key()) +class SqlParameterScalarTypes: + """Supported scalar SQL query parameter types as type objects.""" + + BOOL = ScalarQueryParameterType("BOOL") + BOOLEAN = ScalarQueryParameterType("BOOL") + BIGDECIMAL = ScalarQueryParameterType("BIGNUMERIC") + BIGNUMERIC = ScalarQueryParameterType("BIGNUMERIC") + BYTES = ScalarQueryParameterType("BYTES") + DATE = ScalarQueryParameterType("DATE") + DATETIME = ScalarQueryParameterType("DATETIME") + DECIMAL = ScalarQueryParameterType("NUMERIC") + FLOAT = ScalarQueryParameterType("FLOAT64") + FLOAT64 = ScalarQueryParameterType("FLOAT64") + GEOGRAPHY = ScalarQueryParameterType("GEOGRAPHY") + INT64 = ScalarQueryParameterType("INT64") + INTEGER = ScalarQueryParameterType("INT64") + NUMERIC = ScalarQueryParameterType("NUMERIC") + STRING = ScalarQueryParameterType("STRING") + TIME = ScalarQueryParameterType("TIME") + TIMESTAMP = ScalarQueryParameterType("TIMESTAMP") + + class _QueryResults(object): """Results of a query. diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index cb58ed93f..b52e288f4 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -18,6 +18,7 @@ from typing import Optional from google.cloud.bigquery import standard_sql +from google.cloud.bigquery.enums import StandardSqlTypeNames _DEFAULT_VALUE = object() @@ -27,23 +28,23 @@ # https://cloud.google.com/bigquery/data-types#legacy_sql_data_types # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types LEGACY_TO_STANDARD_TYPES = { - "STRING": standard_sql.StandardSqlDataType.TypeKind.STRING, - "BYTES": standard_sql.StandardSqlDataType.TypeKind.BYTES, - "INTEGER": standard_sql.StandardSqlDataType.TypeKind.INT64, - "INT64": standard_sql.StandardSqlDataType.TypeKind.INT64, - "FLOAT": standard_sql.StandardSqlDataType.TypeKind.FLOAT64, - "FLOAT64": standard_sql.StandardSqlDataType.TypeKind.FLOAT64, - "NUMERIC": standard_sql.StandardSqlDataType.TypeKind.NUMERIC, - "BIGNUMERIC": standard_sql.StandardSqlDataType.TypeKind.BIGNUMERIC, - "BOOLEAN": standard_sql.StandardSqlDataType.TypeKind.BOOL, - "BOOL": standard_sql.StandardSqlDataType.TypeKind.BOOL, - "GEOGRAPHY": standard_sql.StandardSqlDataType.TypeKind.GEOGRAPHY, - "RECORD": standard_sql.StandardSqlDataType.TypeKind.STRUCT, - "STRUCT": standard_sql.StandardSqlDataType.TypeKind.STRUCT, - "TIMESTAMP": standard_sql.StandardSqlDataType.TypeKind.TIMESTAMP, - "DATE": standard_sql.StandardSqlDataType.TypeKind.DATE, - "TIME": standard_sql.StandardSqlDataType.TypeKind.TIME, - "DATETIME": standard_sql.StandardSqlDataType.TypeKind.DATETIME, + "STRING": StandardSqlTypeNames.STRING, + "BYTES": StandardSqlTypeNames.BYTES, + "INTEGER": StandardSqlTypeNames.INT64, + "INT64": StandardSqlTypeNames.INT64, + "FLOAT": StandardSqlTypeNames.FLOAT64, + "FLOAT64": StandardSqlTypeNames.FLOAT64, + "NUMERIC": StandardSqlTypeNames.NUMERIC, + "BIGNUMERIC": StandardSqlTypeNames.BIGNUMERIC, + "BOOLEAN": StandardSqlTypeNames.BOOL, + "BOOL": StandardSqlTypeNames.BOOL, + "GEOGRAPHY": StandardSqlTypeNames.GEOGRAPHY, + "RECORD": StandardSqlTypeNames.STRUCT, + "STRUCT": StandardSqlTypeNames.STRUCT, + "TIMESTAMP": StandardSqlTypeNames.TIMESTAMP, + "DATE": StandardSqlTypeNames.DATE, + "TIME": StandardSqlTypeNames.TIME, + "DATETIME": StandardSqlTypeNames.DATETIME, # no direct conversion from ARRAY, the latter is represented by mode="REPEATED" } """String names of the legacy SQL types to integer codes of Standard SQL standard_sql.""" @@ -290,19 +291,15 @@ def to_standard_sql(self) -> standard_sql.StandardSqlField: sql_type = standard_sql.StandardSqlDataType() if self.mode == "REPEATED": - sql_type.type_kind = standard_sql.StandardSqlDataType.TypeKind.ARRAY + sql_type.type_kind = StandardSqlTypeNames.ARRAY else: sql_type.type_kind = LEGACY_TO_STANDARD_TYPES.get( - self.field_type, - standard_sql.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + self.field_type, StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, ) - if ( - sql_type.type_kind == standard_sql.StandardSqlDataType.TypeKind.ARRAY - ): # noqa: E721 + if sql_type.type_kind == StandardSqlTypeNames.ARRAY: # noqa: E721 array_element_type = LEGACY_TO_STANDARD_TYPES.get( - self.field_type, - standard_sql.StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED, + self.field_type, StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, ) sql_type.array_element_type = standard_sql.StandardSqlDataType( type_kind=array_element_type @@ -310,17 +307,11 @@ def to_standard_sql(self) -> standard_sql.StandardSqlField: # ARRAY cannot directly contain other arrays, only scalar types and STRUCTs # https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#array-type - if ( - array_element_type - == standard_sql.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 - ): + if array_element_type == StandardSqlTypeNames.STRUCT: # noqa: E721 sql_type.array_element_type.struct_type = standard_sql.StandardSqlStructType( fields=(field.to_standard_sql() for field in self.fields) ) - elif ( - sql_type.type_kind - == standard_sql.StandardSqlDataType.TypeKind.STRUCT # noqa: E721 - ): + elif sql_type.type_kind == StandardSqlTypeNames.STRUCT: # noqa: E721 sql_type.struct_type = standard_sql.StandardSqlStructType( fields=(field.to_standard_sql() for field in self.fields) ) diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index 87966ebe2..2b14b1646 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -13,9 +13,10 @@ # limitations under the License. from dataclasses import dataclass -import enum from typing import Any, Dict, Iterable, Optional +from google.cloud.bigquery.enums import StandardSqlTypeNames + @dataclass class StandardSqlDataType: @@ -44,29 +45,9 @@ class StandardSqlDataType: } """ - class TypeKind(str, enum.Enum): - def _generate_next_value_(name, start, count, last_values): - return name - - TYPE_KIND_UNSPECIFIED = enum.auto() - INT64 = enum.auto() - BOOL = enum.auto() - FLOAT64 = enum.auto() - STRING = enum.auto() - BYTES = enum.auto() - TIMESTAMP = enum.auto() - DATE = enum.auto() - TIME = enum.auto() - DATETIME = enum.auto() - INTERVAL = enum.auto() - GEOGRAPHY = enum.auto() - NUMERIC = enum.auto() - BIGNUMERIC = enum.auto() - JSON = enum.auto() - ARRAY = enum.auto() - STRUCT = enum.auto() - - type_kind: Optional[TypeKind] = TypeKind.TYPE_KIND_UNSPECIFIED + type_kind: Optional[ + StandardSqlTypeNames + ] = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED """The top level type of this field. Can be any standard SQL data type, e.g. INT64, DATE, ARRAY. """ @@ -87,19 +68,19 @@ def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this SQL data type.""" if not self.type_kind: - type_kind = self.TypeKind.TYPE_KIND_UNSPECIFIED.value + type_kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED.value else: type_kind = self.type_kind.value result = {"typeKind": type_kind} - if self.type_kind == self.TypeKind.ARRAY: + if self.type_kind == StandardSqlTypeNames.ARRAY: if not self.array_element_type: array_type = None else: array_type = self.array_element_type.to_api_repr() result["arrayElementType"] = array_type - elif self.type_kind == self.TypeKind.STRUCT: + elif self.type_kind == StandardSqlTypeNames.STRUCT: if not self.struct_type: struct_type = None else: @@ -112,11 +93,13 @@ def to_api_repr(self) -> Dict[str, Any]: def from_api_repr(cls, resource: Dict[str, Any]): """Construct an SQL data type instance given its API representation.""" type_kind = resource.get("typeKind") - if type_kind not in cls.TypeKind.__members__: - type_kind = cls.TypeKind.TYPE_KIND_UNSPECIFIED + if type_kind not in StandardSqlTypeNames.__members__: + type_kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED else: # Convert string to an enum member. - type_kind = cls.TypeKind[type_kind] # pytype: disable=missing-parameter + type_kind = StandardSqlTypeNames[ # pytype: disable=missing-parameter + type_kind + ] array_element_type = None if type_kind == cls.type_kind.ARRAY: @@ -124,7 +107,7 @@ def from_api_repr(cls, resource: Dict[str, Any]): array_element_type = cls.from_api_repr(element_type) struct_type = None - if type_kind == cls.TypeKind.STRUCT: + if type_kind == StandardSqlTypeNames.STRUCT: struct_info = resource.get("structType", {}) struct_type = StandardSqlStructType.from_api_repr(struct_info) diff --git a/samples/create_routine.py b/samples/create_routine.py index 3845548ed..8e4f224dc 100644 --- a/samples/create_routine.py +++ b/samples/create_routine.py @@ -33,7 +33,7 @@ def create_routine(routine_id): bigquery.RoutineArgument( name="x", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ], diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 5d4a9f49f..48d7baa5f 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -125,7 +125,7 @@ def routine_id(client, dataset_id): bigquery.RoutineArgument( name="x", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlDataTypeNames.INT64 ), ) ] diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index 490e914b0..482908df9 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -38,21 +38,21 @@ def test_create_routine_ddl(capsys, random_routine_id, client): bigquery.RoutineArgument( name="arr", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.ARRAY, + type_kind=bigquery.StandardSqlDataTypeNames.ARRAY, array_element_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.STRUCT, + type_kind=bigquery.StandardSqlDataTypeNames.STRUCT, struct_type=bigquery.StandardSqlStructType( fields=[ bigquery.StandardSqlField( name="name", type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.STRING + type_kind=bigquery.StandardSqlDataTypeNames.STRING ), ), bigquery.StandardSqlField( name="val", type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlDataTypeNames.INT64 ), ), ] @@ -82,7 +82,7 @@ def test_get_routine(capsys, routine_id): assert "Type: 'SCALAR_FUNCTION'" in out assert "Language: 'SQL'" in out assert "Name: 'x'" in out - assert "type_kind=" in out + assert "type_kind=" in out def test_delete_routine(capsys, routine_id): diff --git a/tests/system/test_client.py b/tests/system/test_client.py index de5e9be1e..4662449c9 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -2181,7 +2181,7 @@ def test_create_routine(self): routine_name = "test_routine" dataset = self.temp_dataset(_make_dataset_id("create_routine")) float64_type = bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.FLOAT64 + type_kind=bigquery.StandardSqlTypeNames.FLOAT64 ) routine = bigquery.Routine( dataset.routine(routine_name), @@ -2196,7 +2196,7 @@ def test_create_routine(self): bigquery.RoutineArgument( name="arr", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataType.TypeKind.ARRAY, + type_kind=bigquery.StandardSqlTypeNames.ARRAY, array_element_type=float64_type, ), ) @@ -2215,14 +2215,19 @@ def test_create_routine(self): assert rows[0].max_value == 100.0 def test_create_tvf_routine(self): - from google.cloud.bigquery import Routine, RoutineArgument, RoutineType + from google.cloud.bigquery import ( + Routine, + RoutineArgument, + RoutineType, + StandardSqlTypeNames, + ) StandardSqlDataType = bigquery.StandardSqlDataType StandardSqlField = bigquery.StandardSqlField StandardSqlTableType = bigquery.StandardSqlTableType - INT64 = StandardSqlDataType.TypeKind.INT64 - STRING = StandardSqlDataType.TypeKind.STRING + INT64 = StandardSqlTypeNames.INT64 + STRING = StandardSqlTypeNames.STRING client = Config.CLIENT diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py index d26528567..ce652d738 100644 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ b/tests/unit/enums/test_standard_sql_data_types.py @@ -30,40 +30,40 @@ def enum_under_test(): @pytest.fixture -def gapic_enum(): +def sql_types_enum(): """The referential autogenerated enum the enum under test is based on.""" - from google.cloud.bigquery.standard_sql import StandardSqlDataType + from google.cloud.bigquery.enums import StandardSqlTypeNames - return StandardSqlDataType.TypeKind + return StandardSqlTypeNames -def test_all_gapic_enum_members_are_known(module_under_test, gapic_enum): - gapic_names = set(type_.name for type_ in gapic_enum) +def test_all_gapic_enum_members_are_known(module_under_test, sql_types_enum): + gapic_names = set(type_.name for type_ in sql_types_enum) anticipated_names = ( module_under_test._SQL_SCALAR_TYPES | module_under_test._SQL_NONSCALAR_TYPES ) assert not (gapic_names - anticipated_names) # no unhandled names -def test_standard_sql_types_enum_members(enum_under_test, gapic_enum): +def test_standard_sql_types_enum_members(enum_under_test, sql_types_enum): # check the presence of a few typical SQL types for name in ("INT64", "FLOAT64", "DATE", "BOOL", "GEOGRAPHY"): assert name in enum_under_test.__members__ # the enum members must match those in the original gapic enum for member in enum_under_test: - assert member.name in gapic_enum.__members__ - assert member.value == gapic_enum[member.name].value + assert member.name in sql_types_enum.__members__ + assert member.value == sql_types_enum[member.name].value # check a few members that should *not* be copied over from the gapic enum for name in ("STRUCT", "ARRAY"): - assert name in gapic_enum.__members__ + assert name in sql_types_enum.__members__ assert name not in enum_under_test.__members__ @pytest.mark.skip(reason="Code generator issue, the docstring is not generated.") def test_standard_sql_types_enum_docstring( - enum_under_test, gapic_enum + enum_under_test, sql_types_enum ): # pragma: NO COVER assert "STRUCT (int):" not in enum_under_test.__doc__ assert "BOOL (int):" in enum_under_test.__doc__ @@ -73,4 +73,4 @@ def test_standard_sql_types_enum_docstring( # except for the header. assert "An Enum of scalar SQL types." in enum_under_test.__doc__ doc_lines = enum_under_test.__doc__.splitlines() - assert set(doc_lines[1:]) <= set(gapic_enum.__doc__.splitlines()) + assert set(doc_lines[1:]) <= set(sql_types_enum.__doc__.splitlines()) diff --git a/tests/unit/routine/test_routine.py b/tests/unit/routine/test_routine.py index c5701d185..80a3def73 100644 --- a/tests/unit/routine/test_routine.py +++ b/tests/unit/routine/test_routine.py @@ -62,14 +62,14 @@ def test_ctor_w_properties(target_class): RoutineArgument( name="x", data_type=bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] body = "x * 3" language = "SQL" return_type = bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ) type_ = "SCALAR_FUNCTION" description = "A routine description." @@ -146,14 +146,14 @@ def test_from_api_repr(target_class): RoutineArgument( name="x", data_type=bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] assert actual_routine.body == "42" assert actual_routine.language == "SQL" assert actual_routine.return_type == bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ) assert actual_routine.return_table_type is None assert actual_routine.type_ == "SCALAR_FUNCTION" @@ -215,7 +215,9 @@ def test_from_api_repr_tvf_function(target_class): assert actual_routine.arguments == [ RoutineArgument( name="a", - data_type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + data_type=StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.INT64 + ), ) ] assert actual_routine.body == "SELECT x FROM UNNEST([1,2,3]) x WHERE x > a" @@ -225,7 +227,7 @@ def test_from_api_repr_tvf_function(target_class): columns=[ StandardSqlField( name="int_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.INT64), ) ] ) @@ -467,11 +469,13 @@ def test_set_return_table_type_w_not_none(object_under_test): columns=[ StandardSqlField( name="int_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.INT64), + type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.INT64), ), StandardSqlField( name="str_col", - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.STRING), + type=StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.STRING + ), ), ] ) diff --git a/tests/unit/routine/test_routine_argument.py b/tests/unit/routine/test_routine_argument.py index 0f1701f95..b7f168a30 100644 --- a/tests/unit/routine/test_routine_argument.py +++ b/tests/unit/routine/test_routine_argument.py @@ -28,7 +28,7 @@ def target_class(): def test_ctor(target_class): data_type = bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ) actual_arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type @@ -51,7 +51,7 @@ def test_from_api_repr(target_class): assert actual_arg.kind == "FIXED_TYPE" assert actual_arg.mode == "IN" assert actual_arg.data_type == bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ) @@ -72,7 +72,7 @@ def test_from_api_repr_w_unknown_fields(target_class): def test_eq(target_class): data_type = bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ) arg = target_class( name="field_name", kind="FIXED_TYPE", mode="IN", data_type=data_type diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index d491ead3f..6125c2103 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1860,7 +1860,7 @@ def test_update_routine(self): RoutineArgument( name="x", data_type=bigquery.standard_sql.StandardSqlDataType( - type_kind=bigquery.standard_sql.StandardSqlDataType.TypeKind.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py index 4afc47b6c..d735771dc 100644 --- a/tests/unit/test_dbapi__helpers.py +++ b/tests/unit/test_dbapi__helpers.py @@ -22,7 +22,7 @@ import pytest import google.cloud._helpers -from google.cloud.bigquery import table, enums +from google.cloud.bigquery import query, table from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions from tests.unit.helpers import _to_pyarrow @@ -338,8 +338,8 @@ def test_custom_on_closed_error_type(self): VALID_BQ_TYPES = [ - (name, getattr(enums.SqlParameterScalarTypes, name)._type) - for name in dir(enums.SqlParameterScalarTypes) + (name, getattr(query.SqlParameterScalarTypes, name)._type) + for name in dir(query.SqlParameterScalarTypes) if not name.startswith("_") ] diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 69a6772e5..71ca67616 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -432,11 +432,11 @@ def test_positional(self): self.assertEqual(param.value, 123) def test_ctor_w_scalar_query_parameter_type(self): - from google.cloud.bigquery import enums + from google.cloud.bigquery import query param = self._make_one( name="foo", - type_=enums.SqlParameterScalarTypes.BIGNUMERIC, + type_=query.SqlParameterScalarTypes.BIGNUMERIC, value=decimal.Decimal("123.456"), ) self.assertEqual(param.name, "foo") diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index a05ca7178..edc05494c 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google.cloud import bigquery from google.cloud.bigquery.standard_sql import StandardSqlStructType from google.cloud.bigquery.schema import PolicyTagList import unittest @@ -205,18 +206,17 @@ def test_fields_property(self): self.assertEqual(schema_field.fields, fields) def test_to_standard_sql_simple_type(self): - sql_type = self._get_standard_sql_data_type_class() examples = ( # a few legacy types - ("INTEGER", sql_type.TypeKind.INT64), - ("FLOAT", sql_type.TypeKind.FLOAT64), - ("BOOLEAN", sql_type.TypeKind.BOOL), - ("DATETIME", sql_type.TypeKind.DATETIME), + ("INTEGER", bigquery.StandardSqlTypeNames.INT64), + ("FLOAT", bigquery.StandardSqlTypeNames.FLOAT64), + ("BOOLEAN", bigquery.StandardSqlTypeNames.BOOL), + ("DATETIME", bigquery.StandardSqlTypeNames.DATETIME), # a few standard types - ("INT64", sql_type.TypeKind.INT64), - ("FLOAT64", sql_type.TypeKind.FLOAT64), - ("BOOL", sql_type.TypeKind.BOOL), - ("GEOGRAPHY", sql_type.TypeKind.GEOGRAPHY), + ("INT64", bigquery.StandardSqlTypeNames.INT64), + ("FLOAT64", bigquery.StandardSqlTypeNames.FLOAT64), + ("BOOL", bigquery.StandardSqlTypeNames.BOOL), + ("GEOGRAPHY", bigquery.StandardSqlTypeNames.GEOGRAPHY), ) for legacy_type, standard_type in examples: field = self._make_one("some_field", legacy_type) @@ -260,31 +260,34 @@ def test_to_standard_sql_struct_type(self): # level 2 fields sub_sub_field_date = standard_sql.StandardSqlField( - name="date_field", type=sql_type(type_kind=sql_type.TypeKind.DATE) + name="date_field", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.DATE), ) sub_sub_field_time = standard_sql.StandardSqlField( - name="time_field", type=sql_type(type_kind=sql_type.TypeKind.TIME) + name="time_field", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.TIME), ) # level 1 fields sub_field_struct = standard_sql.StandardSqlField( name="last_used", type=sql_type( - type_kind=sql_type.TypeKind.STRUCT, + type_kind=bigquery.StandardSqlTypeNames.STRUCT, struct_type=standard_sql.StandardSqlStructType( fields=[sub_sub_field_date, sub_sub_field_time] ), ), ) sub_field_bytes = standard_sql.StandardSqlField( - name="image_content", type=sql_type(type_kind=sql_type.TypeKind.BYTES) + name="image_content", + type=sql_type(type_kind=bigquery.StandardSqlTypeNames.BYTES), ) # level 0 (top level) expected_result = standard_sql.StandardSqlField( name="image_usage", type=sql_type( - type_kind=sql_type.TypeKind.STRUCT, + type_kind=bigquery.StandardSqlTypeNames.STRUCT, struct_type=standard_sql.StandardSqlStructType( fields=[sub_field_bytes, sub_field_struct] ), @@ -313,8 +316,8 @@ def test_to_standard_sql_array_type_simple(self): # construct expected result object expected_sql_type = sql_type( - type_kind=sql_type.TypeKind.ARRAY, - array_element_type=sql_type(type_kind=sql_type.TypeKind.INT64), + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=sql_type(type_kind=bigquery.StandardSqlTypeNames.INT64), ) expected_result = standard_sql.StandardSqlField( name="valid_numbers", type=expected_sql_type @@ -333,22 +336,23 @@ def test_to_standard_sql_array_type_struct(self): # define person STRUCT name_field = standard_sql.StandardSqlField( - name="name", type=sql_type(type_kind=sql_type.TypeKind.STRING) + name="name", type=sql_type(type_kind=bigquery.StandardSqlTypeNames.STRING) ) age_field = standard_sql.StandardSqlField( - name="age", type=sql_type(type_kind=sql_type.TypeKind.INT64) + name="age", type=sql_type(type_kind=bigquery.StandardSqlTypeNames.INT64) ) person_struct = standard_sql.StandardSqlField( name="person_info", type=sql_type( - type_kind=sql_type.TypeKind.STRUCT, + type_kind=bigquery.StandardSqlTypeNames.STRUCT, struct_type=StandardSqlStructType(fields=[name_field, age_field]), ), ) # define expected result - an ARRAY of person structs expected_sql_type = sql_type( - type_kind=sql_type.TypeKind.ARRAY, array_element_type=person_struct.type + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=person_struct.type, ) expected_result = standard_sql.StandardSqlField( name="known_people", type=expected_sql_type @@ -365,14 +369,14 @@ def test_to_standard_sql_array_type_struct(self): self.assertEqual(standard_field, expected_result) def test_to_standard_sql_unknown_type(self): - sql_type = self._get_standard_sql_data_type_class() field = self._make_one("weird_field", "TROOLEAN") standard_field = field.to_standard_sql() self.assertEqual(standard_field.name, "weird_field") self.assertEqual( - standard_field.type.type_kind, sql_type.TypeKind.TYPE_KIND_UNSPECIFIED + standard_field.type.type_kind, + bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, ) def test___eq___wrong_type(self): diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index 5e52e8ecb..2cd18dea1 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -14,6 +14,8 @@ import pytest +from google.cloud import bigquery + class TestStandardSqlDataType: @staticmethod @@ -30,27 +32,26 @@ def test_ctor_w_both_array_element_and_struct_types(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType StandardSqlDataType = self._get_target_class() - TypeKind = StandardSqlDataType.TypeKind + TypeNames = bigquery.StandardSqlTypeNames - array_element_type = StandardSqlDataType(TypeKind.INT64) + array_element_type = StandardSqlDataType(TypeNames.INT64) struct_type = StandardSqlStructType( fields=[ - StandardSqlField("name", StandardSqlDataType(TypeKind.STRING)), - StandardSqlField("age", StandardSqlDataType(TypeKind.INT64)), + StandardSqlField("name", StandardSqlDataType(TypeNames.STRING)), + StandardSqlField("age", StandardSqlDataType(TypeNames.INT64)), ] ) with pytest.raises(ValueError, match=r".*mutally exclusive.*"): self._make_one( - type_kind=TypeKind.TYPE_KIND_UNSPECIFIED, + type_kind=TypeNames.TYPE_KIND_UNSPECIFIED, array_element_type=array_element_type, struct_type=struct_type, ) def test_ctor_default_type_kind(self): - TypeKind = self._get_target_class().TypeKind instance = self._make_one() - assert instance.type_kind == TypeKind.TYPE_KIND_UNSPECIFIED + assert instance.type_kind == bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED def test_to_api_repr_no_type_set(self): instance = self._make_one() @@ -61,16 +62,16 @@ def test_to_api_repr_no_type_set(self): assert result == {"typeKind": "TYPE_KIND_UNSPECIFIED"} def test_to_api_repr_scalar_type(self): - TypeKind = self._get_target_class().TypeKind - instance = self._make_one(TypeKind.FLOAT64) + instance = self._make_one(bigquery.StandardSqlTypeNames.FLOAT64) result = instance.to_api_repr() assert result == {"typeKind": "FLOAT64"} def test_to_api_repr_array_type_element_type_missing(self): - TypeKind = self._get_target_class().TypeKind - instance = self._make_one(TypeKind.ARRAY, array_element_type=None) + instance = self._make_one( + bigquery.StandardSqlTypeNames.ARRAY, array_element_type=None + ) result = instance.to_api_repr() @@ -78,10 +79,12 @@ def test_to_api_repr_array_type_element_type_missing(self): assert result == expected def test_to_api_repr_array_type_w_element_type(self): - TypeKind = self._get_target_class().TypeKind - - array_element_type = self._make_one(type_kind=TypeKind.BOOL) - instance = self._make_one(TypeKind.ARRAY, array_element_type=array_element_type) + array_element_type = self._make_one( + type_kind=bigquery.StandardSqlTypeNames.BOOL + ) + instance = self._make_one( + bigquery.StandardSqlTypeNames.ARRAY, array_element_type=array_element_type + ) result = instance.to_api_repr() @@ -89,8 +92,9 @@ def test_to_api_repr_array_type_w_element_type(self): assert result == expected def test_to_api_repr_struct_type_field_types_missing(self): - TypeKind = self._get_target_class().TypeKind - instance = self._make_one(TypeKind.STRUCT, struct_type=None) + instance = self._make_one( + bigquery.StandardSqlTypeNames.STRUCT, struct_type=None + ) result = instance.to_api_repr() @@ -101,28 +105,28 @@ def test_to_api_repr_struct_type_w_field_types(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType StandardSqlDataType = self._get_target_class() - TypeKind = StandardSqlDataType.TypeKind + TypeNames = bigquery.StandardSqlTypeNames person_type = StandardSqlStructType( fields=[ - StandardSqlField("name", StandardSqlDataType(TypeKind.STRING)), - StandardSqlField("age", StandardSqlDataType(TypeKind.INT64)), + StandardSqlField("name", StandardSqlDataType(TypeNames.STRING)), + StandardSqlField("age", StandardSqlDataType(TypeNames.INT64)), ] ) employee_type = StandardSqlStructType( fields=[ - StandardSqlField("job_title", StandardSqlDataType(TypeKind.STRING)), - StandardSqlField("salary", StandardSqlDataType(TypeKind.FLOAT64)), + StandardSqlField("job_title", StandardSqlDataType(TypeNames.STRING)), + StandardSqlField("salary", StandardSqlDataType(TypeNames.FLOAT64)), StandardSqlField( "employee_info", StandardSqlDataType( - type_kind=TypeKind.STRUCT, struct_type=person_type, + type_kind=TypeNames.STRUCT, struct_type=person_type, ), ), ] ) - instance = self._make_one(TypeKind.STRUCT, struct_type=employee_type) + instance = self._make_one(TypeNames.STRUCT, struct_type=employee_type) result = instance.to_api_repr() expected = { @@ -153,7 +157,7 @@ def test_from_api_repr_empty_resource(self): result = klass.from_api_repr(resource={}) expected = klass( - type_kind=klass.TypeKind.TYPE_KIND_UNSPECIFIED, + type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, array_element_type=None, struct_type=None, ) @@ -166,7 +170,9 @@ def test_from_api_repr_scalar_type(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=klass.TypeKind.DATE, array_element_type=None, struct_type=None, + type_kind=bigquery.StandardSqlTypeNames.DATE, + array_element_type=None, + struct_type=None, ) assert result == expected @@ -177,8 +183,8 @@ def test_from_api_repr_array_type_full(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=klass.TypeKind.ARRAY, - array_element_type=klass(type_kind=klass.TypeKind.BYTES), + type_kind=bigquery.StandardSqlTypeNames.ARRAY, + array_element_type=klass(type_kind=bigquery.StandardSqlTypeNames.BYTES), struct_type=None, ) assert result == expected @@ -190,9 +196,9 @@ def test_from_api_repr_array_type_missing_element_type(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=klass.TypeKind.ARRAY, + type_kind=bigquery.StandardSqlTypeNames.ARRAY, array_element_type=klass( - type_kind=klass.TypeKind.TYPE_KIND_UNSPECIFIED, + type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, array_element_type=None, struct_type=None, ), @@ -205,7 +211,7 @@ def test_from_api_repr_struct_type_nested(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() - TypeKind = klass.TypeKind + TypeNames = bigquery.StandardSqlTypeNames resource = { "typeKind": "STRUCT", @@ -232,19 +238,19 @@ def test_from_api_repr_struct_type_nested(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=TypeKind.STRUCT, + type_kind=TypeNames.STRUCT, struct_type=StandardSqlStructType( fields=[ - StandardSqlField("job_title", klass(TypeKind.STRING)), - StandardSqlField("salary", klass(TypeKind.FLOAT64)), + StandardSqlField("job_title", klass(TypeNames.STRING)), + StandardSqlField("salary", klass(TypeNames.FLOAT64)), StandardSqlField( "employee_info", klass( - type_kind=TypeKind.STRUCT, + type_kind=TypeNames.STRUCT, struct_type=StandardSqlStructType( fields=[ - StandardSqlField("name", klass(TypeKind.STRING)), - StandardSqlField("age", klass(TypeKind.INT64)), + StandardSqlField("name", klass(TypeNames.STRING)), + StandardSqlField("age", klass(TypeNames.INT64)), ] ), ), @@ -263,7 +269,7 @@ def test_from_api_repr_struct_type_missing_struct_info(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=klass.TypeKind.STRUCT, + type_kind=bigquery.StandardSqlTypeNames.STRUCT, array_element_type=None, struct_type=StandardSqlStructType(fields=[]), ) @@ -274,7 +280,7 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() - TypeKind = klass.TypeKind + TypeNames = bigquery.StandardSqlTypeNames resource = { "typeKind": "STRUCT", @@ -289,11 +295,11 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=TypeKind.STRUCT, + type_kind=TypeNames.STRUCT, struct_type=StandardSqlStructType( fields=[ - StandardSqlField(None, klass(TypeKind.STRING)), - StandardSqlField("salary", klass(TypeKind.TYPE_KIND_UNSPECIFIED)), + StandardSqlField(None, klass(TypeNames.STRING)), + StandardSqlField("salary", klass(TypeNames.TYPE_KIND_UNSPECIFIED)), ] ), ) @@ -365,14 +371,14 @@ def test_from_api_repr_with_incomplete_columns(self): expected = StandardSqlField( name=None, - type=StandardSqlDataType(type_kind=StandardSqlDataType.TypeKind.BOOL), + type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.BOOL), ) assert result.columns[0] == expected expected = StandardSqlField( name="bar", type=StandardSqlDataType( - type_kind=StandardSqlDataType.TypeKind.TYPE_KIND_UNSPECIFIED + type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED ), ) assert result.columns[1] == expected From b2d8dcac4917c4334b6ed0ef4d374b1ae34c1e0f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 6 Sep 2021 19:40:20 +0200 Subject: [PATCH 16/32] Disable pytype false positive --- google/cloud/bigquery/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index ff803c819..6e86135f6 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -224,7 +224,7 @@ def _make_sql_scalars_enum(): new_enum = enum.Enum( "StandardSqlDataTypes", - ( + ( # pytype: disable=missing-parameter (member.name, member.value) for member in StandardSqlTypeNames if member.name in _SQL_SCALAR_TYPES From ced79f490efed7627137616f9fb3378445899cee Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 7 Sep 2021 10:31:42 +0200 Subject: [PATCH 17/32] Rename StandardSqlDataTypes to StandardSqlScalarTypes --- google/cloud/bigquery/__init__.py | 4 ++-- google/cloud/bigquery/enums.py | 4 ++-- tests/unit/enums/test_standard_sql_data_types.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 0b6968c65..d520fbfbd 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -41,7 +41,7 @@ from google.cloud.bigquery.enums import DecimalTargetType from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.enums import SqlTypeNames -from google.cloud.bigquery.enums import StandardSqlDataTypes +from google.cloud.bigquery.enums import StandardSqlScalarTypes from google.cloud.bigquery.enums import StandardSqlTypeNames from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.external_config import BigtableOptions @@ -180,7 +180,7 @@ "SchemaUpdateOption", "SourceFormat", "SqlTypeNames", - "StandardSqlDataTypes", + "StandardSqlScalarTypes", "StandardSqlTypeNames", "WriteDisposition", # EncryptionConfiguration diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 6e86135f6..68bdcd129 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -223,7 +223,7 @@ def _make_sql_scalars_enum(): """Create an enum based on the types enum containing only SQL scalar types.""" new_enum = enum.Enum( - "StandardSqlDataTypes", + "StandardSqlScalarTypes", ( # pytype: disable=missing-parameter (member.name, member.value) for member in StandardSqlTypeNames @@ -235,7 +235,7 @@ def _make_sql_scalars_enum(): return new_enum -StandardSqlDataTypes = _make_sql_scalars_enum() +StandardSqlScalarTypes = _make_sql_scalars_enum() # See also: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py index ce652d738..8342aa0ed 100644 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ b/tests/unit/enums/test_standard_sql_data_types.py @@ -24,9 +24,9 @@ def module_under_test(): @pytest.fixture def enum_under_test(): - from google.cloud.bigquery.enums import StandardSqlDataTypes + from google.cloud.bigquery.enums import StandardSqlScalarTypes - return StandardSqlDataTypes + return StandardSqlScalarTypes @pytest.fixture From 34c382a81e30db2a2796f3d0097934902ecd562e Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 7 Sep 2021 14:47:55 +0200 Subject: [PATCH 18/32] Remove scalar SQL types enum A decision was made that it is not all that useful and can be removed. --- google/cloud/bigquery/__init__.py | 2 - google/cloud/bigquery/enums.py | 41 ---------- tests/unit/enums/__init__.py | 13 ---- .../enums/test_standard_sql_data_types.py | 76 ------------------- tests/unit/gapic/__init__.py | 15 ---- 5 files changed, 147 deletions(-) delete mode 100644 tests/unit/enums/__init__.py delete mode 100644 tests/unit/enums/test_standard_sql_data_types.py delete mode 100644 tests/unit/gapic/__init__.py diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index d520fbfbd..660a660b4 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -41,7 +41,6 @@ from google.cloud.bigquery.enums import DecimalTargetType from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.enums import SqlTypeNames -from google.cloud.bigquery.enums import StandardSqlScalarTypes from google.cloud.bigquery.enums import StandardSqlTypeNames from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.external_config import BigtableOptions @@ -180,7 +179,6 @@ "SchemaUpdateOption", "SourceFormat", "SqlTypeNames", - "StandardSqlScalarTypes", "StandardSqlTypeNames", "WriteDisposition", # EncryptionConfiguration diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 68bdcd129..cecdaa503 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -197,47 +197,6 @@ def _generate_next_value_(name, start, count, last_values): STRUCT = enum.auto() -_SQL_SCALAR_TYPES = frozenset( - ( - "INT64", - "BOOL", - "FLOAT64", - "STRING", - "BYTES", - "TIMESTAMP", - "DATE", - "TIME", - "DATETIME", - "INTERVAL", - "GEOGRAPHY", - "NUMERIC", - "BIGNUMERIC", - "JSON", - ) -) - -_SQL_NONSCALAR_TYPES = frozenset(("TYPE_KIND_UNSPECIFIED", "ARRAY", "STRUCT")) - - -def _make_sql_scalars_enum(): - """Create an enum based on the types enum containing only SQL scalar types.""" - - new_enum = enum.Enum( - "StandardSqlScalarTypes", - ( # pytype: disable=missing-parameter - (member.name, member.value) - for member in StandardSqlTypeNames - if member.name in _SQL_SCALAR_TYPES - ), - ) - new_enum.__doc__ = "Scalar standard SQL types." - - return new_enum - - -StandardSqlScalarTypes = _make_sql_scalars_enum() - - # See also: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types # and https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types class SqlTypeNames(str, enum.Enum): diff --git a/tests/unit/enums/__init__.py b/tests/unit/enums/__init__.py deleted file mode 100644 index c5cce0430..000000000 --- a/tests/unit/enums/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019, Google LLC All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/unit/enums/test_standard_sql_data_types.py b/tests/unit/enums/test_standard_sql_data_types.py deleted file mode 100644 index 8342aa0ed..000000000 --- a/tests/unit/enums/test_standard_sql_data_types.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - - -@pytest.fixture -def module_under_test(): - from google.cloud.bigquery import enums - - return enums - - -@pytest.fixture -def enum_under_test(): - from google.cloud.bigquery.enums import StandardSqlScalarTypes - - return StandardSqlScalarTypes - - -@pytest.fixture -def sql_types_enum(): - """The referential autogenerated enum the enum under test is based on.""" - from google.cloud.bigquery.enums import StandardSqlTypeNames - - return StandardSqlTypeNames - - -def test_all_gapic_enum_members_are_known(module_under_test, sql_types_enum): - gapic_names = set(type_.name for type_ in sql_types_enum) - anticipated_names = ( - module_under_test._SQL_SCALAR_TYPES | module_under_test._SQL_NONSCALAR_TYPES - ) - assert not (gapic_names - anticipated_names) # no unhandled names - - -def test_standard_sql_types_enum_members(enum_under_test, sql_types_enum): - # check the presence of a few typical SQL types - for name in ("INT64", "FLOAT64", "DATE", "BOOL", "GEOGRAPHY"): - assert name in enum_under_test.__members__ - - # the enum members must match those in the original gapic enum - for member in enum_under_test: - assert member.name in sql_types_enum.__members__ - assert member.value == sql_types_enum[member.name].value - - # check a few members that should *not* be copied over from the gapic enum - for name in ("STRUCT", "ARRAY"): - assert name in sql_types_enum.__members__ - assert name not in enum_under_test.__members__ - - -@pytest.mark.skip(reason="Code generator issue, the docstring is not generated.") -def test_standard_sql_types_enum_docstring( - enum_under_test, sql_types_enum -): # pragma: NO COVER - assert "STRUCT (int):" not in enum_under_test.__doc__ - assert "BOOL (int):" in enum_under_test.__doc__ - assert "TIME (int):" in enum_under_test.__doc__ - - # All lines in the docstring should actually come from the original docstring, - # except for the header. - assert "An Enum of scalar SQL types." in enum_under_test.__doc__ - doc_lines = enum_under_test.__doc__.splitlines() - assert set(doc_lines[1:]) <= set(sql_types_enum.__doc__.splitlines()) diff --git a/tests/unit/gapic/__init__.py b/tests/unit/gapic/__init__.py deleted file mode 100644 index 4de65971c..000000000 --- a/tests/unit/gapic/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# From af45ebcd3c2b5ad1cf38bad8dac95f9d125f1338 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 7 Sep 2021 17:12:03 +0200 Subject: [PATCH 19/32] Fix mis-renamed enum references in samples tests --- samples/tests/conftest.py | 2 +- samples/tests/test_routine_samples.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 48d7baa5f..4764a571f 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -125,7 +125,7 @@ def routine_id(client, dataset_id): bigquery.RoutineArgument( name="x", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataTypeNames.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ) ] diff --git a/samples/tests/test_routine_samples.py b/samples/tests/test_routine_samples.py index 482908df9..91518cece 100644 --- a/samples/tests/test_routine_samples.py +++ b/samples/tests/test_routine_samples.py @@ -38,21 +38,21 @@ def test_create_routine_ddl(capsys, random_routine_id, client): bigquery.RoutineArgument( name="arr", data_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataTypeNames.ARRAY, + type_kind=bigquery.StandardSqlTypeNames.ARRAY, array_element_type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataTypeNames.STRUCT, + type_kind=bigquery.StandardSqlTypeNames.STRUCT, struct_type=bigquery.StandardSqlStructType( fields=[ bigquery.StandardSqlField( name="name", type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataTypeNames.STRING + type_kind=bigquery.StandardSqlTypeNames.STRING ), ), bigquery.StandardSqlField( name="val", type=bigquery.StandardSqlDataType( - type_kind=bigquery.StandardSqlDataTypeNames.INT64 + type_kind=bigquery.StandardSqlTypeNames.INT64 ), ), ] From 6739d37892d42f6bbaec7abe62b01ba0e7981df6 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 17 Sep 2021 09:51:52 -0500 Subject: [PATCH 20/32] remove unused generated code loop --- owlbot.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/owlbot.py b/owlbot.py index 2756b7873..ea2d355b6 100644 --- a/owlbot.py +++ b/owlbot.py @@ -21,32 +21,6 @@ common = gcp.CommonTemplates() -default_version = "v2" - -for library in s.get_staging_dirs(default_version): - s.move( - library, - excludes=[ - "*.tar.gz", - ".coveragerc", - "docs/index.rst", - f"docs/bigquery_{library.name}/*_service.rst", - f"docs/bigquery_{library.name}/services.rst", - "README.rst", - "noxfile.py", - "setup.py", - f"scripts/fixup_bigquery_{library.name}_keywords.py", - "google/cloud/bigquery/__init__.py", - "google/cloud/bigquery/py.typed", - # The library does not use any protos (it implements its own type classes), - # thus omit the generated code altogether. - f"google/cloud/bigquery_{library.name}/**", - f"tests/unit/gapic/bigquery_{library.name}/**", - ], - ) - -s.remove_staging_dirs() - # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- From fc31ceaad8805002b8f49a0ccc118bea148a3121 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 17 Sep 2021 14:57:07 +0000 Subject: [PATCH 21/32] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- docs/conf.py | 2 ++ setup.cfg | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index b8ddbd8c8..82d36a3cd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -363,6 +363,8 @@ "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), + "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None), "geopandas": ("https://geopandas.org/", None), } diff --git a/setup.cfg b/setup.cfg index 8eefc4435..28b7b0f26 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ inputs = google/cloud/ exclude = tests/ - google/cloud/bigquery_v2/ output = .pytype/ disable = # There's some issue with finding some pyi files, thus disabling. From 6f0c55f30dbd940b1b9fc7a2628eb7962dca88ae Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 17 Sep 2021 10:23:45 -0500 Subject: [PATCH 22/32] omit value != 0 --- google/cloud/bigquery/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index d5a31d6af..592259bf0 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -119,7 +119,7 @@ def created(self) -> Optional[datetime.datetime]: Read-only. """ value = self._properties.get("creationTime") - if value is not None and value != 0: + if value is not None: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) @@ -132,7 +132,7 @@ def modified(self) -> Optional[datetime.datetime]: Read-only. """ value = value = self._properties.get("lastModifiedTime") - if value is not None and value != 0: + if value is not None: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) From 0c986b3fb53ae3da52c2fc68ebc42a16804e7380 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 17 Sep 2021 10:36:14 -0500 Subject: [PATCH 23/32] remove hack for startTime --- google/cloud/bigquery/model.py | 12 ------------ tests/unit/model/test_model.py | 6 +++--- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 592259bf0..ead1d561c 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -21,7 +21,6 @@ from typing import Any, Dict, Optional, Sequence, Union import google.cloud._helpers -from google.api_core import datetime_helpers from google.cloud.bigquery import _helpers from google.cloud.bigquery import standard_sql from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration @@ -261,19 +260,8 @@ def from_api_repr(cls, resource: Dict[str, Any]) -> "Model": Model parsed from ``resource``. """ this = cls(None) - resource = copy.deepcopy(resource) this._properties = resource - - # Convert from millis-from-epoch to timestamp well-known type. - # TODO: Remove this hack once CL 238585470 hits prod. - for training_run in resource.get("trainingRuns", ()): - start_time = training_run.get("startTime") - if not start_time or "-" in start_time: # Already right format? - continue - start_time = datetime_helpers.from_microseconds(1e3 * float(start_time)) - training_run["startTime"] = datetime_helpers.to_rfc3339(start_time) - return this def _build_resource(self, filter_fields): diff --git a/tests/unit/model/test_model.py b/tests/unit/model/test_model.py index 9b99ca5d5..c5f9b77c1 100644 --- a/tests/unit/model/test_model.py +++ b/tests/unit/model/test_model.py @@ -94,9 +94,9 @@ def test_from_api_repr(target_class): }, { "trainingOptions": {"initialLearnRate": 0.25}, - # Allow milliseconds since epoch format. - # TODO: Remove this hack once CL 238585470 hits prod. - "startTime": str(google.cloud._helpers._millis(expiration_time)), + "startTime": str( + google.cloud._helpers._datetime_to_rfc3339(expiration_time) + ), }, ], "featureColumns": [], From cfba949adcfe567617273c7682fbc235361e279e Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 20 Sep 2021 16:26:16 +0200 Subject: [PATCH 24/32] Permanently remove unneded intersphinx links --- owlbot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/owlbot.py b/owlbot.py index ea2d355b6..be493957e 100644 --- a/owlbot.py +++ b/owlbot.py @@ -52,6 +52,10 @@ ], ) +# Remove unneeded intersphinx links, the library does not use any proto-generated code. +s.replace("docs/conf.py", r'\s+"(proto-plus|protobuf)":.*$', "") + + # ---------------------------------------------------------------------------- # Samples templates # ---------------------------------------------------------------------------- From b497784f825026398afd65e8e17fbceaa01c1e8a Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 20 Sep 2021 14:28:57 +0000 Subject: [PATCH 25/32] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- docs/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 82d36a3cd..b8ddbd8c8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -363,8 +363,6 @@ "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), - "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), - "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None), "geopandas": ("https://geopandas.org/", None), } From 8cd235db66607d3071272da5fe90c62ecb33057f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 22 Sep 2021 15:35:37 +0200 Subject: [PATCH 26/32] Refactor SQL type classes to use _properties --- google/cloud/bigquery/standard_sql.py | 335 ++++++++++++++++++++------ tests/unit/test_standard_sql_types.py | 4 +- 2 files changed, 262 insertions(+), 77 deletions(-) diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index 2b14b1646..e032116b6 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass -from typing import Any, Dict, Iterable, Optional +import copy +from typing import Any, Dict, Generator, Iterable, List, Optional from google.cloud.bigquery.enums import StandardSqlTypeNames -@dataclass class StandardSqlDataType: """The type of a variable, e.g., a function argument. @@ -43,52 +42,137 @@ class StandardSqlDataType: ] } } - """ - type_kind: Optional[ - StandardSqlTypeNames - ] = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED - """The top level type of this field. Can be any standard SQL data type, - e.g. INT64, DATE, ARRAY. + Args: + type_kind: + The top level type of this field. Can be any standard SQL data type, + e.g. INT64, DATE, ARRAY. + array_element_type: + The type of the array's elements, if type_kind is ARRAY. + struct_type: + The fields of this struct, in order, if type_kind is STRUCT. """ - array_element_type: Optional["StandardSqlDataType"] = None - """The type of the array's elements, if type_kind is ARRAY.""" - - struct_type: Optional["StandardSqlStructType"] = None - """The fields of this struct, in order, if type_kind is STRUCT.""" - - def __post_init__(self): - if self.array_element_type is not None and self.struct_type is not None: + def __init__( + self, + type_kind: Optional[ + StandardSqlTypeNames + ] = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, + array_element_type: Optional["StandardSqlDataType"] = None, + struct_type: Optional["StandardSqlStructType"] = None, + ): + if array_element_type is not None and struct_type is not None: raise ValueError( "array_element_type and struct_type are mutally exclusive." ) - def to_api_repr(self) -> Dict[str, Any]: - """Construct the API resource representation of this SQL data type.""" - - if not self.type_kind: - type_kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED.value + self._properties = {} + + self.type_kind = type_kind + self.array_element_type = array_element_type + self.struct_type = struct_type + + @property + def type_kind(self) -> StandardSqlTypeNames: + """The top level type of this field. + + Can be any standard SQL data type, e.g. INT64, DATE, ARRAY. + """ + kind = self._properties["typeKind"] + return StandardSqlTypeNames[kind] # pytype: disable=missing-parameter + + @type_kind.setter + def type_kind(self, value: Optional[StandardSqlTypeNames]): + new_instance = "typeKind" not in self._properties + + if not new_instance: + current = self._properties.get("typeKind") + + if value == current: + return # Nothing to change. + + if ( + current == StandardSqlTypeNames.ARRAY + and self.array_element_type.type_kind + is not StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + ): + raise ValueError( + "Cannot change {StandardSqlTypeNames.ARRAY} type_kind, first set " + "array_element_type attribute to None." + ) + + if current == StandardSqlTypeNames.STRUCT and self.struct_type.fields: + raise ValueError( + "Cannot change {StandardSqlTypeNames.STRUCT} type_kind, first set " + "struct_type attribute to None." + ) + + if not value: + kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED.value else: - type_kind = self.type_kind.value - - result = {"typeKind": type_kind} + kind = value.value + self._properties["typeKind"] = kind + @property + def array_element_type(self) -> Optional["StandardSqlDataType"]: + """The type of the array's elements, if type_kind is ARRAY.""" + result = None if self.type_kind == StandardSqlTypeNames.ARRAY: - if not self.array_element_type: - array_type = None + element_type = self._properties.get("arrayElementType") + if element_type is not None: + result = StandardSqlDataType.from_api_repr(element_type) else: - array_type = self.array_element_type.to_api_repr() - result["arrayElementType"] = array_type - elif self.type_kind == StandardSqlTypeNames.STRUCT: - if not self.struct_type: - struct_type = None + result = StandardSqlDataType() + + return result + + @array_element_type.setter + def array_element_type(self, value: Optional["StandardSqlDataType"]): + if self.type_kind != StandardSqlTypeNames.ARRAY and value is not None: + raise ValueError( + "Cannot set to a non-None value if type_kind is not " + f"{StandardSqlTypeNames.ARRAY.value}." + ) + + element_type = None if value is None else value.to_api_repr() + + if element_type is None: + self._properties.pop("arrayElementType", None) + else: + self._properties["arrayElementType"] = element_type + + @property + def struct_type(self) -> Optional["StandardSqlStructType"]: + """The fields of this struct, in order, if type_kind is STRUCT.""" + result = None + if self.type_kind == StandardSqlTypeNames.STRUCT: + struct_info = self._properties.get("structType") + if struct_info is not None: + result = StandardSqlStructType.from_api_repr(struct_info) else: - struct_type = self.struct_type.to_api_repr() - result["structType"] = struct_type + result = StandardSqlStructType() return result + @struct_type.setter + def struct_type(self, value: Optional["StandardSqlStructType"]): + if self.type_kind != StandardSqlTypeNames.STRUCT and value is not None: + raise ValueError( + "Cannot set to a non-None value if type_kind is not " + f"{StandardSqlTypeNames.STRUCT.value}." + ) + + struct_type = None if value is None else value.to_api_repr() + + if struct_type is None: + self._properties.pop("structType", None) + else: + self._properties["structType"] = struct_type + + def to_api_repr(self) -> Dict[str, Any]: + """Construct the API resource representation of this SQL data type.""" + return copy.deepcopy(self._properties) + @classmethod def from_api_repr(cls, resource: Dict[str, Any]): """Construct an SQL data type instance given its API representation.""" @@ -102,40 +186,86 @@ def from_api_repr(cls, resource: Dict[str, Any]): ] array_element_type = None - if type_kind == cls.type_kind.ARRAY: - element_type = resource.get("arrayElementType", {}) - array_element_type = cls.from_api_repr(element_type) + if type_kind == StandardSqlTypeNames.ARRAY: + element_type = resource.get("arrayElementType") + if element_type: + array_element_type = cls.from_api_repr(element_type) struct_type = None if type_kind == StandardSqlTypeNames.STRUCT: - struct_info = resource.get("structType", {}) - struct_type = StandardSqlStructType.from_api_repr(struct_info) + struct_info = resource.get("structType") + if struct_info: + struct_type = StandardSqlStructType.from_api_repr(struct_info) return cls(type_kind, array_element_type, struct_type) + def __eq__(self, other): + if not isinstance(other, StandardSqlDataType): + return NotImplemented + else: + return ( + self.type_kind == other.type_kind + and self.array_element_type == other.array_element_type + and self.struct_type == other.struct_type + ) + + __hash__ = None + -@dataclass class StandardSqlField: """A field or a column. See: https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlField - """ - - name: Optional[str] = None - """The name of this field. Can be absent for struct fields.""" - type: Optional["StandardSqlDataType"] = None - """The type of this parameter. Absent if not explicitly specified. + Args: + name: + The name of this field. Can be absent for struct fields. + type: + The type of this parameter. Absent if not explicitly specified. - For example, CREATE FUNCTION statement can omit the return type; in this case the - output parameter does not have this "type" field). + For example, CREATE FUNCTION statement can omit the return type; in this + case the output parameter does not have this "type" field). """ + def __init__( + self, name: Optional[str] = None, type: Optional[StandardSqlDataType] = None + ): + if type is not None: + type = type.to_api_repr() + + self._properties = {"name": name, "type": type} + + @property + def name(self): + """The name of this field. Can be absent for struct fields.""" + return self._properties["name"] + + @name.setter + def name(self, value: Optional[str]): + self._properties["name"] = value + + @property + def type(self): + """The type of this parameter. Absent if not explicitly specified. + + For example, CREATE FUNCTION statement can omit the return type; in this + case the output parameter does not have this "type" field). + """ + result = self._properties["type"] + if result is not None: + result = StandardSqlDataType.from_api_repr(result) + return result + + @type.setter + def type(self, value: Optional[StandardSqlDataType]): + if value is not None: + value = value.to_api_repr() + self._properties["type"] = value + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this SQL field.""" - type_repr = None if self.type is None else self.type.to_api_repr() - return {"name": self.name, "type": type_repr} + return copy.deepcopy(self._properties) @classmethod def from_api_repr(cls, resource: Dict[str, Any]): @@ -146,62 +276,117 @@ def from_api_repr(cls, resource: Dict[str, Any]): ) return result + def __eq__(self, other): + if not isinstance(other, StandardSqlField): + return NotImplemented + else: + return self.name == other.name and self.type == other.type + + __hash__ = None + -@dataclass(init=False) class StandardSqlStructType: """Type of a struct field. See: https://cloud.google.com/bigquery/docs/reference/rest/v2/StandardSqlDataType#StandardSqlStructType + + Args: + fields: The fields in this struct. """ - fields: Optional[Iterable["StandardSqlField"]] + def __init__(self, fields: Optional[Iterable[StandardSqlField]] = None): + if fields is None: + fields = [] + self._properties = {"fields": [field.to_api_repr() for field in fields]} - def __init__(self, fields=None): - self.fields = [] if fields is None else [field for field in fields] + @property + def fields(self) -> List[StandardSqlField]: + """The fields in this struct.""" + result = self._fields_from_resource(self._properties) + return list(result) + + @fields.setter + def fields(self, value: Iterable[StandardSqlField]): + self._properties["fields"] = [field.to_api_repr() for field in value] def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this SQL struct type.""" - fields = [field.to_api_repr() for field in self.fields] - result = {"fields": fields} - return result + return copy.deepcopy(self._properties) @classmethod def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlStructType": """Construct an SQL struct type instance given its API representation.""" - fields = ( - StandardSqlField.from_api_repr(field) - for field in resource.get("fields", []) - ) + fields = cls._fields_from_resource(resource) return cls(fields=fields) + @staticmethod + def _fields_from_resource( + resource: Dict[str, Any] + ) -> Generator[StandardSqlField, None, None]: + """Yield field instancess based on the resource info.""" + for field_resource in resource.get("fields", []): + yield StandardSqlField.from_api_repr(field_resource) + + def __eq__(self, other): + if not isinstance(other, StandardSqlStructType): + return NotImplemented + else: + return self.fields == other.fields + + __hash__ = None + -@dataclass(init=False) class StandardSqlTableType: """A table type. + See: https://cloud.google.com/workflows/docs/reference/googleapis/bigquery/v2/Overview#StandardSqlTableType - """ - columns: Iterable[StandardSqlField] - """The columns in this table type""" + Args: + columns: The columns in this table type. + """ def __init__(self, columns: Iterable[StandardSqlField]): - self.columns = [col for col in columns] + self._properties = {"columns": [col.to_api_repr() for col in columns]} + + @property + def columns(self) -> List[StandardSqlField]: + """The columns in this table type.""" + return list(self._columns_from_resource(self._properties)) + + @columns.setter + def columns(self, value: Iterable[StandardSqlField]): + self._properties["columns"] = [col.to_api_repr() for col in value] def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this SQL table type.""" - columns = [col.to_api_repr() for col in self.columns] - return {"columns": columns} + return copy.deepcopy(self._properties) @classmethod def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlTableType": """Construct an SQL table type instance given its API representation.""" - columns = ( - StandardSqlField( - name=column.get("name"), - type=StandardSqlDataType.from_api_repr(column.get("type", {})), - ) - for column in resource.get("columns", []) - ) + columns = cls._columns_from_resource(resource) return cls(columns=columns) + + @staticmethod + def _columns_from_resource( + resource: Dict[str, Any] + ) -> Generator[StandardSqlField, None, None]: + """Yield column instances based on the resource info.""" + for column in resource.get("columns", []): + type_ = column.get("type") + if type_ is None: + type_ = {} + + yield StandardSqlField( + name=column.get("name"), type=StandardSqlDataType.from_api_repr(type_), + ) + + def __eq__(self, other): + if not isinstance(other, StandardSqlTableType): + return NotImplemented + else: + return self.columns == other.columns + + __hash__ = None diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index 2cd18dea1..f13ee45da 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -75,7 +75,7 @@ def test_to_api_repr_array_type_element_type_missing(self): result = instance.to_api_repr() - expected = {"typeKind": "ARRAY", "arrayElementType": None} + expected = {"typeKind": "ARRAY"} assert result == expected def test_to_api_repr_array_type_w_element_type(self): @@ -98,7 +98,7 @@ def test_to_api_repr_struct_type_field_types_missing(self): result = instance.to_api_repr() - assert result == {"typeKind": "STRUCT", "structType": None} + assert result == {"typeKind": "STRUCT"} def test_to_api_repr_struct_type_w_field_types(self): from google.cloud.bigquery.standard_sql import StandardSqlField From 1b29ac214bf1626283235688cba33af5b792d849 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 23 Sep 2021 13:19:41 +0200 Subject: [PATCH 27/32] Remove validation logic from StandardSqlDataType Client-side validation logic can drift away from the server's, thus we let the backend perform all validation of submitted request data. --- google/cloud/bigquery/standard_sql.py | 67 ++++----------------------- tests/unit/test_standard_sql_types.py | 34 +------------- 2 files changed, 12 insertions(+), 89 deletions(-) diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index e032116b6..15a4fca72 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -61,11 +61,6 @@ def __init__( array_element_type: Optional["StandardSqlDataType"] = None, struct_type: Optional["StandardSqlStructType"] = None, ): - if array_element_type is not None and struct_type is not None: - raise ValueError( - "array_element_type and struct_type are mutally exclusive." - ) - self._properties = {} self.type_kind = type_kind @@ -83,30 +78,6 @@ def type_kind(self) -> StandardSqlTypeNames: @type_kind.setter def type_kind(self, value: Optional[StandardSqlTypeNames]): - new_instance = "typeKind" not in self._properties - - if not new_instance: - current = self._properties.get("typeKind") - - if value == current: - return # Nothing to change. - - if ( - current == StandardSqlTypeNames.ARRAY - and self.array_element_type.type_kind - is not StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED - ): - raise ValueError( - "Cannot change {StandardSqlTypeNames.ARRAY} type_kind, first set " - "array_element_type attribute to None." - ) - - if current == StandardSqlTypeNames.STRUCT and self.struct_type.fields: - raise ValueError( - "Cannot change {StandardSqlTypeNames.STRUCT} type_kind, first set " - "struct_type attribute to None." - ) - if not value: kind = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED.value else: @@ -116,24 +87,15 @@ def type_kind(self, value: Optional[StandardSqlTypeNames]): @property def array_element_type(self) -> Optional["StandardSqlDataType"]: """The type of the array's elements, if type_kind is ARRAY.""" - result = None - if self.type_kind == StandardSqlTypeNames.ARRAY: - element_type = self._properties.get("arrayElementType") - if element_type is not None: - result = StandardSqlDataType.from_api_repr(element_type) - else: - result = StandardSqlDataType() + element_type = self._properties.get("arrayElementType") - return result + if element_type is not None: + return StandardSqlDataType.from_api_repr(element_type) + else: + return None @array_element_type.setter def array_element_type(self, value: Optional["StandardSqlDataType"]): - if self.type_kind != StandardSqlTypeNames.ARRAY and value is not None: - raise ValueError( - "Cannot set to a non-None value if type_kind is not " - f"{StandardSqlTypeNames.ARRAY.value}." - ) - element_type = None if value is None else value.to_api_repr() if element_type is None: @@ -144,24 +106,15 @@ def array_element_type(self, value: Optional["StandardSqlDataType"]): @property def struct_type(self) -> Optional["StandardSqlStructType"]: """The fields of this struct, in order, if type_kind is STRUCT.""" - result = None - if self.type_kind == StandardSqlTypeNames.STRUCT: - struct_info = self._properties.get("structType") - if struct_info is not None: - result = StandardSqlStructType.from_api_repr(struct_info) - else: - result = StandardSqlStructType() + struct_info = self._properties.get("structType") - return result + if struct_info is not None: + return StandardSqlStructType.from_api_repr(struct_info) + else: + return None @struct_type.setter def struct_type(self, value: Optional["StandardSqlStructType"]): - if self.type_kind != StandardSqlTypeNames.STRUCT and value is not None: - raise ValueError( - "Cannot set to a non-None value if type_kind is not " - f"{StandardSqlTypeNames.STRUCT.value}." - ) - struct_type = None if value is None else value.to_api_repr() if struct_type is None: diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index f13ee45da..4ff33b2e8 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from google.cloud import bigquery @@ -27,28 +25,6 @@ def _get_target_class(): def _make_one(self, *args, **kw): return self._get_target_class()(*args, **kw) - def test_ctor_w_both_array_element_and_struct_types(self): - from google.cloud.bigquery.standard_sql import StandardSqlField - from google.cloud.bigquery.standard_sql import StandardSqlStructType - - StandardSqlDataType = self._get_target_class() - TypeNames = bigquery.StandardSqlTypeNames - - array_element_type = StandardSqlDataType(TypeNames.INT64) - struct_type = StandardSqlStructType( - fields=[ - StandardSqlField("name", StandardSqlDataType(TypeNames.STRING)), - StandardSqlField("age", StandardSqlDataType(TypeNames.INT64)), - ] - ) - - with pytest.raises(ValueError, match=r".*mutally exclusive.*"): - self._make_one( - type_kind=TypeNames.TYPE_KIND_UNSPECIFIED, - array_element_type=array_element_type, - struct_type=struct_type, - ) - def test_ctor_default_type_kind(self): instance = self._make_one() assert instance.type_kind == bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED @@ -197,11 +173,7 @@ def test_from_api_repr_array_type_missing_element_type(self): expected = klass( type_kind=bigquery.StandardSqlTypeNames.ARRAY, - array_element_type=klass( - type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, - array_element_type=None, - struct_type=None, - ), + array_element_type=None, struct_type=None, ) assert result == expected @@ -261,8 +233,6 @@ def test_from_api_repr_struct_type_nested(self): assert result == expected def test_from_api_repr_struct_type_missing_struct_info(self): - from google.cloud.bigquery.standard_sql import StandardSqlStructType - klass = self._get_target_class() resource = {"typeKind": "STRUCT"} @@ -271,7 +241,7 @@ def test_from_api_repr_struct_type_missing_struct_info(self): expected = klass( type_kind=bigquery.StandardSqlTypeNames.STRUCT, array_element_type=None, - struct_type=StandardSqlStructType(fields=[]), + struct_type=None, ) assert result == expected From adcaa03a0e6b329e383b7528871cc49a1b6dc08f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 23 Sep 2021 14:39:54 +0200 Subject: [PATCH 28/32] Refactor SQL models to use shared state Modifying a sub-type should be automatically reflected on the parent type referencing that subtype. --- google/cloud/bigquery/standard_sql.py | 84 ++++++++++++++++----------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index 15a4fca72..0784acec4 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -13,7 +13,7 @@ # limitations under the License. import copy -from typing import Any, Dict, Generator, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional from google.cloud.bigquery.enums import StandardSqlTypeNames @@ -89,11 +89,13 @@ def array_element_type(self) -> Optional["StandardSqlDataType"]: """The type of the array's elements, if type_kind is ARRAY.""" element_type = self._properties.get("arrayElementType") - if element_type is not None: - return StandardSqlDataType.from_api_repr(element_type) - else: + if element_type is None: return None + result = StandardSqlDataType() + result._properties = element_type # We do not use a copy on purpose. + return result + @array_element_type.setter def array_element_type(self, value: Optional["StandardSqlDataType"]): element_type = None if value is None else value.to_api_repr() @@ -108,11 +110,13 @@ def struct_type(self) -> Optional["StandardSqlStructType"]: """The fields of this struct, in order, if type_kind is STRUCT.""" struct_info = self._properties.get("structType") - if struct_info is not None: - return StandardSqlStructType.from_api_repr(struct_info) - else: + if struct_info is None: return None + result = StandardSqlStructType() + result._properties = struct_info # We do not use a copy on purpose. + return result + @struct_type.setter def struct_type(self, value: Optional["StandardSqlStructType"]): struct_type = None if value is None else value.to_api_repr() @@ -190,7 +194,7 @@ def __init__( self._properties = {"name": name, "type": type} @property - def name(self): + def name(self) -> Optional[str]: """The name of this field. Can be absent for struct fields.""" return self._properties["name"] @@ -199,15 +203,19 @@ def name(self, value: Optional[str]): self._properties["name"] = value @property - def type(self): + def type(self) -> Optional[StandardSqlDataType]: """The type of this parameter. Absent if not explicitly specified. For example, CREATE FUNCTION statement can omit the return type; in this case the output parameter does not have this "type" field). """ - result = self._properties["type"] - if result is not None: - result = StandardSqlDataType.from_api_repr(result) + type_info = self._properties["type"] + + if type_info is None: + return None + + result = StandardSqlDataType() + result._properties = type_info # We do not use a copy on purpose. return result @type.setter @@ -256,8 +264,14 @@ def __init__(self, fields: Optional[Iterable[StandardSqlField]] = None): @property def fields(self) -> List[StandardSqlField]: """The fields in this struct.""" - result = self._fields_from_resource(self._properties) - return list(result) + result = [] + + for field_resource in self._properties.get("fields", []): + field = StandardSqlField() + field._properties = field_resource # We do not use a copy on purpose. + result.append(field) + + return result @fields.setter def fields(self, value: Iterable[StandardSqlField]): @@ -270,17 +284,12 @@ def to_api_repr(self) -> Dict[str, Any]: @classmethod def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlStructType": """Construct an SQL struct type instance given its API representation.""" - fields = cls._fields_from_resource(resource) + fields = ( + StandardSqlField.from_api_repr(field_resource) + for field_resource in resource.get("fields", []) + ) return cls(fields=fields) - @staticmethod - def _fields_from_resource( - resource: Dict[str, Any] - ) -> Generator[StandardSqlField, None, None]: - """Yield field instancess based on the resource info.""" - for field_resource in resource.get("fields", []): - yield StandardSqlField.from_api_repr(field_resource) - def __eq__(self, other): if not isinstance(other, StandardSqlStructType): return NotImplemented @@ -306,7 +315,14 @@ def __init__(self, columns: Iterable[StandardSqlField]): @property def columns(self) -> List[StandardSqlField]: """The columns in this table type.""" - return list(self._columns_from_resource(self._properties)) + result = [] + + for column_resource in self._properties.get("columns", []): + column = StandardSqlField() + column._properties = column_resource # We do not use a copy on purpose. + result.append(column) + + return result @columns.setter def columns(self, value: Iterable[StandardSqlField]): @@ -319,22 +335,20 @@ def to_api_repr(self) -> Dict[str, Any]: @classmethod def from_api_repr(cls, resource: Dict[str, Any]) -> "StandardSqlTableType": """Construct an SQL table type instance given its API representation.""" - columns = cls._columns_from_resource(resource) - return cls(columns=columns) + columns = [] - @staticmethod - def _columns_from_resource( - resource: Dict[str, Any] - ) -> Generator[StandardSqlField, None, None]: - """Yield column instances based on the resource info.""" - for column in resource.get("columns", []): - type_ = column.get("type") + for column_resource in resource.get("columns", []): + type_ = column_resource.get("type") if type_ is None: type_ = {} - yield StandardSqlField( - name=column.get("name"), type=StandardSqlDataType.from_api_repr(type_), + column = StandardSqlField( + name=column_resource.get("name"), + type=StandardSqlDataType.from_api_repr(type_), ) + columns.append(column) + + return cls(columns=columns) def __eq__(self, other): if not isinstance(other, StandardSqlTableType): From 3db7ff3ef44d87b869bcb55efe7b87e50d7bd680 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 23 Sep 2021 16:28:06 +0200 Subject: [PATCH 29/32] Add more informative string repr of SQL type --- google/cloud/bigquery/standard_sql.py | 4 ++++ tests/unit/test_standard_sql_types.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index 0784acec4..479929c74 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -168,6 +168,10 @@ def __eq__(self, other): __hash__ = None + def __str__(self): + result = f"{self.__class__.__name__}(type_kind={self.type_kind!r}, ...)" + return result + class StandardSqlField: """A field or a column. diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index 4ff33b2e8..1922f9c84 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -275,6 +275,11 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): ) assert result == expected + def test_str(self): + instance = self._make_one(type_kind=bigquery.StandardSqlTypeNames.BOOL) + bool_type_repr = repr(bigquery.StandardSqlTypeNames.BOOL) + assert str(instance) == f"StandardSqlDataType(type_kind={bool_type_repr}, ...)" + class TestStandardSqlTableType: @staticmethod From cf562ffee7eb7c4c851557704629df2f8b38e247 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 23 Sep 2021 17:38:05 +0200 Subject: [PATCH 30/32] Bump test coverage to 100% --- tests/unit/test_standard_sql_types.py | 279 +++++++++++++++++++++++--- 1 file changed, 254 insertions(+), 25 deletions(-) diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index 1922f9c84..b91f877cc 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.cloud import bigquery +from unittest import mock + +import pytest + +from google.cloud import bigquery as bq class TestStandardSqlDataType: @@ -27,7 +31,7 @@ def _make_one(self, *args, **kw): def test_ctor_default_type_kind(self): instance = self._make_one() - assert instance.type_kind == bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + assert instance.type_kind == bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED def test_to_api_repr_no_type_set(self): instance = self._make_one() @@ -38,7 +42,7 @@ def test_to_api_repr_no_type_set(self): assert result == {"typeKind": "TYPE_KIND_UNSPECIFIED"} def test_to_api_repr_scalar_type(self): - instance = self._make_one(bigquery.StandardSqlTypeNames.FLOAT64) + instance = self._make_one(bq.StandardSqlTypeNames.FLOAT64) result = instance.to_api_repr() @@ -46,7 +50,7 @@ def test_to_api_repr_scalar_type(self): def test_to_api_repr_array_type_element_type_missing(self): instance = self._make_one( - bigquery.StandardSqlTypeNames.ARRAY, array_element_type=None + bq.StandardSqlTypeNames.ARRAY, array_element_type=None ) result = instance.to_api_repr() @@ -55,11 +59,9 @@ def test_to_api_repr_array_type_element_type_missing(self): assert result == expected def test_to_api_repr_array_type_w_element_type(self): - array_element_type = self._make_one( - type_kind=bigquery.StandardSqlTypeNames.BOOL - ) + array_element_type = self._make_one(type_kind=bq.StandardSqlTypeNames.BOOL) instance = self._make_one( - bigquery.StandardSqlTypeNames.ARRAY, array_element_type=array_element_type + bq.StandardSqlTypeNames.ARRAY, array_element_type=array_element_type ) result = instance.to_api_repr() @@ -68,9 +70,7 @@ def test_to_api_repr_array_type_w_element_type(self): assert result == expected def test_to_api_repr_struct_type_field_types_missing(self): - instance = self._make_one( - bigquery.StandardSqlTypeNames.STRUCT, struct_type=None - ) + instance = self._make_one(bq.StandardSqlTypeNames.STRUCT, struct_type=None) result = instance.to_api_repr() @@ -81,7 +81,7 @@ def test_to_api_repr_struct_type_w_field_types(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType StandardSqlDataType = self._get_target_class() - TypeNames = bigquery.StandardSqlTypeNames + TypeNames = bq.StandardSqlTypeNames person_type = StandardSqlStructType( fields=[ @@ -133,7 +133,7 @@ def test_from_api_repr_empty_resource(self): result = klass.from_api_repr(resource={}) expected = klass( - type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, + type_kind=bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, array_element_type=None, struct_type=None, ) @@ -146,7 +146,7 @@ def test_from_api_repr_scalar_type(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=bigquery.StandardSqlTypeNames.DATE, + type_kind=bq.StandardSqlTypeNames.DATE, array_element_type=None, struct_type=None, ) @@ -159,8 +159,8 @@ def test_from_api_repr_array_type_full(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=bigquery.StandardSqlTypeNames.ARRAY, - array_element_type=klass(type_kind=bigquery.StandardSqlTypeNames.BYTES), + type_kind=bq.StandardSqlTypeNames.ARRAY, + array_element_type=klass(type_kind=bq.StandardSqlTypeNames.BYTES), struct_type=None, ) assert result == expected @@ -172,7 +172,7 @@ def test_from_api_repr_array_type_missing_element_type(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=bigquery.StandardSqlTypeNames.ARRAY, + type_kind=bq.StandardSqlTypeNames.ARRAY, array_element_type=None, struct_type=None, ) @@ -183,7 +183,7 @@ def test_from_api_repr_struct_type_nested(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() - TypeNames = bigquery.StandardSqlTypeNames + TypeNames = bq.StandardSqlTypeNames resource = { "typeKind": "STRUCT", @@ -239,7 +239,7 @@ def test_from_api_repr_struct_type_missing_struct_info(self): result = klass.from_api_repr(resource=resource) expected = klass( - type_kind=bigquery.StandardSqlTypeNames.STRUCT, + type_kind=bq.StandardSqlTypeNames.STRUCT, array_element_type=None, struct_type=None, ) @@ -250,7 +250,7 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): from google.cloud.bigquery.standard_sql import StandardSqlStructType klass = self._get_target_class() - TypeNames = bigquery.StandardSqlTypeNames + TypeNames = bq.StandardSqlTypeNames resource = { "typeKind": "STRUCT", @@ -275,12 +275,198 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): ) assert result == expected + def test__eq__another_type(self): + instance = self._make_one() + + class SqlTypeWannabe: + pass + + not_a_type = SqlTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one() + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "type_kind": bq.StandardSqlTypeNames.GEOGRAPHY, + "array_element_type": bq.StandardSqlDataType( + type_kind=bq.StandardSqlTypeNames.INT64 + ), + "struct_type": bq.StandardSqlStructType(fields=[]), + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + @pytest.mark.parametrize( + ("attr_name", "value", "value2"), + ( + ( + "type_kind", + bq.StandardSqlTypeNames.INT64, + bq.StandardSqlTypeNames.FLOAT64, + ), + ( + "array_element_type", + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.STRING), + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ), + ( + "struct_type", + bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="foo")]), + bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="bar")]), + ), + ), + ) + def test__eq__attribute_differs(self, attr_name, value, value2): + instance = self._make_one(**{attr_name: value}) + instance2 = self._make_one(**{attr_name: value2}) + assert instance != instance2 + def test_str(self): - instance = self._make_one(type_kind=bigquery.StandardSqlTypeNames.BOOL) - bool_type_repr = repr(bigquery.StandardSqlTypeNames.BOOL) + instance = self._make_one(type_kind=bq.StandardSqlTypeNames.BOOL) + bool_type_repr = repr(bq.StandardSqlTypeNames.BOOL) assert str(instance) == f"StandardSqlDataType(type_kind={bool_type_repr}, ...)" +class TestStandardSqlField: + # This class only contains minimum tests to cover what other tests don't + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlField + + return StandardSqlField + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_name(self): + instance = self._make_one(name="foo") + assert instance.name == "foo" + instance.name = "bar" + assert instance.name == "bar" + + def test_type_missing(self): + instance = self._make_one(type=None) + assert instance.type is None + + def test_type_set_none(self): + instance = self._make_one( + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL) + ) + instance.type = None + assert instance.type is None + + def test_type_set_not_none(self): + instance = self._make_one(type=bq.StandardSqlDataType(type_kind=None)) + instance.type = bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INT64) + assert instance.type == bq.StandardSqlDataType( + type_kind=bq.StandardSqlTypeNames.INT64 + ) + + def test__eq__another_type(self): + instance = self._make_one( + name="foo", + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ) + + class FieldWannabe: + pass + + not_a_field = FieldWannabe() + not_a_field._properties = instance._properties + + assert instance != not_a_field # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one( + name="foo", + type=bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), + ) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "name": "foo", + "type": bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INT64), + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + @pytest.mark.parametrize( + ("attr_name", "value", "value2"), + ( + ("name", "foo", "bar",), + ( + "type", + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.INTERVAL), + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.TIME), + ), + ), + ) + def test__eq__attribute_differs(self, attr_name, value, value2): + instance = self._make_one(**{attr_name: value}) + instance2 = self._make_one(**{attr_name: value2}) + assert instance != instance2 + + +class TestStandardSqlStructType: + # This class only contains minimum tests to cover what other tests don't + + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.standard_sql import StandardSqlStructType + + return StandardSqlStructType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_fields(self): + instance = self._make_one(fields=[]) + assert instance.fields == [] + + new_fields = [bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + instance.fields = new_fields + assert instance.fields == new_fields + + def test__eq__another_type(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + + class StructTypeWannabe: + pass + + not_a_type = StructTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "fields": [bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + def test__eq__attribute_differs(self): + instance = self._make_one(fields=[bq.StandardSqlField(name="foo")]) + instance2 = self._make_one( + fields=[bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + ) + assert instance != instance2 + + class TestStandardSqlTableType: @staticmethod def _get_target_class(): @@ -306,6 +492,17 @@ def test_columns_shallow_copy(self): columns.pop() assert len(instance.columns) == 3 # Still the same. + def test_columns_setter(self): + from google.cloud.bigquery.standard_sql import StandardSqlField + + columns = [StandardSqlField("foo")] + instance = self._make_one(columns=columns) + assert instance.columns == columns + + new_columns = [StandardSqlField(name="bar")] + instance.columns = new_columns + assert instance.columns == new_columns + def test_to_api_repr_no_columns(self): instance = self._make_one(columns=[]) result = instance.to_api_repr() @@ -345,15 +542,47 @@ def test_from_api_repr_with_incomplete_columns(self): assert len(result.columns) == 2 expected = StandardSqlField( - name=None, - type=StandardSqlDataType(type_kind=bigquery.StandardSqlTypeNames.BOOL), + name=None, type=StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.BOOL), ) assert result.columns[0] == expected expected = StandardSqlField( name="bar", type=StandardSqlDataType( - type_kind=bigquery.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED + type_kind=bq.StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED ), ) assert result.columns[1] == expected + + def test__eq__another_type(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + + class TableTypeWannabe: + pass + + not_a_type = TableTypeWannabe() + not_a_type._properties = instance._properties + + assert instance != not_a_type # Can't fake it. + + def test__eq__delegates_comparison_to_another_type(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + assert instance == mock.ANY + + def test__eq__similar_instance(self): + kwargs = { + "columns": [ + bq.StandardSqlField(name="foo"), + bq.StandardSqlField(name="bar"), + ] + } + instance = self._make_one(**kwargs) + instance2 = self._make_one(**kwargs) + assert instance == instance2 + + def test__eq__attribute_differs(self): + instance = self._make_one(columns=[bq.StandardSqlField(name="foo")]) + instance2 = self._make_one( + columns=[bq.StandardSqlField(name="foo"), bq.StandardSqlField(name="bar")] + ) + assert instance != instance2 From 026c05d9818426fea36354e90652b5f3a120e213 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 24 Sep 2021 09:33:33 +0200 Subject: [PATCH 31/32] Remove unneeded dataclasses dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 37a3b4254..130d8f49c 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "dataclasses >= 0.8, < 1.0; python_version < '3.7'", # Until we drop Python 3.6 support. "grpcio >= 1.38.1, < 2.0dev", # https://github.com/googleapis/python-bigquery/issues/695 # NOTE: Maintainers, please do not require google-api-core>=2.x.x # Until this issue is closed From 01a02ff199f75134015221292491bdc31934d563 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 27 Sep 2021 16:25:00 -0500 Subject: [PATCH 32/32] Update google/cloud/bigquery/model.py --- google/cloud/bigquery/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index ead1d561c..18b7b13ec 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -149,6 +149,9 @@ def model_type(self) -> str: def training_runs(self) -> Sequence[Dict[str, Any]]: """Information for all training runs in increasing order of start time. + Dictionaries are in REST API format. See: + https://cloud.google.com/bigquery/docs/reference/rest/v2/models#trainingrun + Read-only. """ return self._properties.get("trainingRuns", [])