diff --git a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index 175e5f47f1fb..5ec974e19ef5 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -16,10 +16,13 @@ package com.google.cloud.bigquery; +import com.google.api.core.ApiFunction; import com.google.api.services.bigquery.model.JobConfiguration; import com.google.api.services.bigquery.model.JobStatistics2; import com.google.api.services.bigquery.model.JobStatistics3; import com.google.api.services.bigquery.model.JobStatistics4; +import com.google.cloud.StringEnumType; +import com.google.cloud.StringEnumValue; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.Lists; @@ -311,6 +314,7 @@ static LoadStatistics fromPb(com.google.api.services.bigquery.model.JobStatistic } } + /** * A Google BigQuery Query Job statistics. */ @@ -320,18 +324,92 @@ public static class QueryStatistics extends JobStatistics { private final Integer billingTier; private final Boolean cacheHit; + private final String ddlOperationPerformed; + private final TableId ddlTargetTable; + private final Long estimatedBytesProcessed; + private final Long numDmlAffectedRows; + private final List referencedTables; + private final StatementType statementType; private final Long totalBytesBilled; private final Long totalBytesProcessed; + private final Long totalPartitionsProcessed; + private final Long totalSlotMs; private final List queryPlan; private final List timeline; private final Schema schema; + + /** + * StatementType represents possible types of SQL statements reported as part of the + * QueryStatistics of a BigQuery job. + */ + public static final class StatementType extends StringEnumValue { + private static final long serialVersionUID = 818920627219751204L; + + private static final ApiFunction CONSTRUCTOR = + new ApiFunction() { + @Override + public StatementType apply(String constant) { + return new StatementType(constant); + } + }; + + private static final StringEnumType type = new StringEnumType( + StatementType.class, + CONSTRUCTOR); + + public static final StatementType SELECT = type.createAndRegister("SELECT"); + public static final StatementType UPDATE = type.createAndRegister("UPDATE"); + public static final StatementType INSERT = type.createAndRegister("INSERT"); + public static final StatementType DELETE = type.createAndRegister("DELETE"); + public static final StatementType CREATE_TABLE = type.createAndRegister("CREATE_TABLE"); + public static final StatementType CREATE_TABLE_AS_SELECT = type.createAndRegister("CREATE_TABLE_AS_SELECT"); + public static final StatementType CREATE_VIEW = type.createAndRegister("CREATE_VIEW"); + public static final StatementType DROP_TABLE = type.createAndRegister("DROP_TABLE"); + public static final StatementType DROP_VIEW = type.createAndRegister("DROP_VIEW"); + public static final StatementType MERGE = type.createAndRegister("MERGE"); + + private StatementType(String constant) { + super(constant); + } + + /** + * Get the StatementType for the given String constant, and throw an exception if the constant is + * not recognized. + */ + public static StatementType valueOfStrict(String constant) { + return type.valueOfStrict(constant); + } + + /** + * Get the State for the given String constant, and allow unrecognized values. + */ + public static StatementType valueOf(String constant) { + return type.valueOf(constant); + } + + /** + * Return the known values for State. + */ + public static StatementType[] values() { + return type.values(); + } + } + static final class Builder extends JobStatistics.Builder { private Integer billingTier; private Boolean cacheHit; + private String ddlOperationPerformed; + private TableId ddlTargetTable; + private Long estimatedBytesProcessed; + private Long numDmlAffectedRows; + private List referencedTables; + private StatementType statementType; private Long totalBytesBilled; private Long totalBytesProcessed; + private Long totalPartitionsProcessed; + private Long totalSlotMs; private List queryPlan; private List timeline; private Schema schema; @@ -343,8 +421,25 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP if (statisticsPb.getQuery() != null) { this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); + this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); + if (statisticsPb.getQuery().getDdlTargetTable() != null) { + this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable()); + } + this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed(); + this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); + this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed(); + this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs(); + if (statisticsPb.getQuery().getStatementType() != null) { + this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType()); + } + + if (statisticsPb.getQuery().getReferencedTables() != null) { + this.referencedTables = + Lists.transform( + statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION); + } if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform( @@ -371,6 +466,41 @@ Builder setCacheHit(Boolean cacheHit) { return self(); } + Builder setDDLOperationPerformed(String ddlOperationPerformed) { + this.ddlOperationPerformed = ddlOperationPerformed; + return self(); + } + + Builder setDDLTargetTable(TableId ddlTargetTable) { + this.ddlTargetTable = ddlTargetTable; + return self(); + } + + Builder setEstimatedBytesProcessed(Long estimatedBytesProcessed) { + this.estimatedBytesProcessed = estimatedBytesProcessed; + return self(); + } + + Builder setNumDmlAffectedRows(Long numDmlAffectedRows) { + this.numDmlAffectedRows = numDmlAffectedRows; + return self(); + } + + Builder setReferenceTables(List referencedTables) { + this.referencedTables = referencedTables; + return self(); + } + + Builder setStatementType(StatementType statementType) { + this.statementType = statementType; + return self(); + } + + Builder setStatementType(String strStatementType) { + this.statementType = StatementType.valueOf(strStatementType); + return self(); + } + Builder setTotalBytesBilled(Long totalBytesBilled) { this.totalBytesBilled = totalBytesBilled; return self(); @@ -381,6 +511,16 @@ Builder setTotalBytesProcessed(Long totalBytesProcessed) { return self(); } + Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) { + this.totalPartitionsProcessed = totalPartitionsProcessed; + return self(); + } + + Builder setTotalSlotMs(Long totalSlotMs) { + this.totalSlotMs = totalSlotMs; + return self(); + } + Builder setQueryPlan(List queryPlan) { this.queryPlan = queryPlan; return self(); @@ -406,8 +546,16 @@ private QueryStatistics(Builder builder) { super(builder); this.billingTier = builder.billingTier; this.cacheHit = builder.cacheHit; + this.ddlOperationPerformed = builder.ddlOperationPerformed; + this.ddlTargetTable = builder.ddlTargetTable; + this.estimatedBytesProcessed = builder.estimatedBytesProcessed; + this.numDmlAffectedRows = builder.numDmlAffectedRows; + this.referencedTables = builder.referencedTables; + this.statementType = builder.statementType; this.totalBytesBilled = builder.totalBytesBilled; this.totalBytesProcessed = builder.totalBytesProcessed; + this.totalPartitionsProcessed = builder.totalPartitionsProcessed; + this.totalSlotMs = builder.totalSlotMs; this.queryPlan = builder.queryPlan; this.timeline = builder.timeline; this.schema = builder.schema; @@ -432,6 +580,47 @@ public Boolean getCacheHit() { return cacheHit; } + /** + * [BETA] For DDL queries, returns the operation applied to the DDL target table. + */ + public String getDdlOperationPerformed() { return ddlOperationPerformed; } + + /** + * [BETA] For DDL queries, returns the TableID of the targeted table. + */ + public TableId getDdlTargetTable() { return ddlTargetTable; } + + /** + * The original estimate of bytes processed for the job. + */ + public Long getEstimatedBytesProcessed() { return estimatedBytesProcessed; } + + /** + * The number of rows affected by a DML statement. + * Present only for DML statements INSERT, UPDATE or DELETE. + */ + public Long getNumDmlAffectedRows() { return numDmlAffectedRows; } + + /** + * Referenced tables for the job. + * Queries that reference more than 50 tables will not have a complete list. + */ + public List getReferencedTables() { return referencedTables; } + + /** + * [BETA] The type of query statement, if valid. + * Possible values include: + * SELECT + * INSERT + * UPDATE + * DELETE + * CREATE_TABLE + * CREATE_TABLE_AS_SELECT + * DROP_TABLE + * CREATE_VIEW + * DROP_VIEW + */ + public StatementType getStatementType() { return statementType; } /** * Returns the total number of bytes billed for the job. @@ -448,6 +637,15 @@ public Long getTotalBytesProcessed() { return totalBytesProcessed; } + /** + * Total number of partitions processed from all partitioned tables referenced in the job. + */ + public Long getTotalPartitionsProcessed() { return totalPartitionsProcessed; } + + /** + * Returns the slot-milliseconds consumed by the query. + */ + public Long getTotalSlotMs() { return totalSlotMs; } /** * Returns the query plan as a list of stages or {@code null} if a query plan is not available. @@ -509,8 +707,22 @@ com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); + queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed); + queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); + queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed); + queryStatisticsPb.setTotalSlotMs(totalSlotMs); + if (ddlTargetTable != null) { + queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb()); + } + + if (referencedTables != null) { + queryStatisticsPb.setReferencedTables(Lists.transform(referencedTables, TableId.TO_PB_FUNCTION)); + } + if (statementType != null) { + queryStatisticsPb.setStatementType(statementType.toString()); + } if (queryPlan != null) { queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); } diff --git a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java index be738c89b355..cae861be6d85 100644 --- a/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java +++ b/google-cloud-clients/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java @@ -32,12 +32,22 @@ public class JobStatisticsTest { private static final Integer BILLING_TIER = 42; private static final Boolean CACHE_HIT = true; + private static final String DDL_OPERATION_PERFORMED = "SKIP"; + private static final TableId DDL_TARGET_TABLE = TableId.of("foo", "bar", "baz"); + private static final Long ESTIMATE_BYTES_PROCESSED = 101L; + private static final Long NUM_DML_AFFECTED_ROWS = 88L; + private static final QueryStatistics.StatementType STATEMENT_TYPE = QueryStatistics.StatementType.SELECT; private static final Long TOTAL_BYTES_BILLED = 24L; private static final Long TOTAL_BYTES_PROCESSED = 42L; + private static final Long TOTAL_PARTITION_PROCESSED = 63L; + private static final Long TOTAL_SLOT_MS = 10202L; private static final Long INPUT_BYTES = 1L; private static final Long INPUT_FILES = 2L; private static final Long OUTPUT_BYTES = 3L; private static final Long OUTPUT_ROWS = 4L; + private static final List REFERENCED_TABLES = ImmutableList.of( + TableId.of("foo", "bar", "table1"), + TableId.of("foo","bar","table2")); private static final List FILE_COUNT = ImmutableList.of(1L, 2L, 3L); private static final Long CREATION_TIME = 10L; private static final Long END_TIME = 20L; @@ -112,8 +122,16 @@ public class JobStatisticsTest { .setStartTime(START_TIME) .setBillingTier(BILLING_TIER) .setCacheHit(CACHE_HIT) + .setDDLOperationPerformed(DDL_OPERATION_PERFORMED) + .setDDLTargetTable(DDL_TARGET_TABLE) + .setEstimatedBytesProcessed(ESTIMATE_BYTES_PROCESSED) + .setNumDmlAffectedRows(NUM_DML_AFFECTED_ROWS) + .setReferenceTables(REFERENCED_TABLES) + .setStatementType(STATEMENT_TYPE) .setTotalBytesBilled(TOTAL_BYTES_BILLED) .setTotalBytesProcessed(TOTAL_BYTES_PROCESSED) + .setTotalPartitionsProcessed(TOTAL_PARTITION_PROCESSED) + .setTotalSlotMs(TOTAL_SLOT_MS) .setQueryPlan(QUERY_PLAN) .setTimeline(TIMELINE) .setSchema(SCHEMA) @@ -146,9 +164,16 @@ public void testBuilder() { assertEquals(END_TIME, QUERY_STATISTICS.getEndTime()); assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier()); assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit()); + assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed()); + assertEquals(DDL_TARGET_TABLE, QUERY_STATISTICS.getDdlTargetTable()); + assertEquals(ESTIMATE_BYTES_PROCESSED, QUERY_STATISTICS.getEstimatedBytesProcessed()); + assertEquals(NUM_DML_AFFECTED_ROWS, QUERY_STATISTICS.getNumDmlAffectedRows()); + assertEquals(REFERENCED_TABLES, QUERY_STATISTICS.getReferencedTables()); + assertEquals(STATEMENT_TYPE, QUERY_STATISTICS.getStatementType()); assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled()); assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed()); - assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed()); + assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed()); + assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs()); assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan()); assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline()); @@ -165,8 +190,15 @@ public void testBuilder() { assertEquals(END_TIME, QUERY_STATISTICS_INCOMPLETE.getEndTime()); assertEquals(BILLING_TIER, QUERY_STATISTICS_INCOMPLETE.getBillingTier()); assertEquals(CACHE_HIT, QUERY_STATISTICS_INCOMPLETE.getCacheHit()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlOperationPerformed()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlTargetTable()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getEstimatedBytesProcessed()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getNumDmlAffectedRows()); assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesBilled()); assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesProcessed()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalPartitionsProcessed()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalSlotMs()); + assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getReferencedTables()); assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getQueryPlan()); } @@ -235,10 +267,18 @@ private void compareQueryStatistics(QueryStatistics expected, QueryStatistics va compareStatistics(expected, value); assertEquals(expected.getBillingTier(), value.getBillingTier()); assertEquals(expected.getCacheHit(), value.getCacheHit()); + assertEquals(expected.getDdlOperationPerformed(), value.getDdlOperationPerformed()); + assertEquals(expected.getDdlTargetTable(), value.getDdlTargetTable()); + assertEquals(expected.getEstimatedBytesProcessed(), value.getEstimatedBytesProcessed()); assertEquals(expected.getTotalBytesBilled(), value.getTotalBytesBilled()); assertEquals(expected.getTotalBytesProcessed(), value.getTotalBytesProcessed()); + assertEquals(expected.getTotalPartitionsProcessed(), value.getTotalPartitionsProcessed()); + assertEquals(expected.getTotalSlotMs(), value.getTotalSlotMs()); assertEquals(expected.getQueryPlan(), value.getQueryPlan()); + assertEquals(expected.getReferencedTables(), value.getReferencedTables()); assertEquals(expected.getSchema(), value.getSchema()); + assertEquals(expected.getStatementType(), value.getStatementType()); + assertEquals(expected.getTimeline(), value.getTimeline()); } private void compareStatistics(JobStatistics expected, JobStatistics value) {