Skip to content

BigQuery: add missing query statistics. #3253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shollyman merged 4 commits into from
May 22, 2018
Merged

BigQuery: add missing query statistics. #3253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82bcff3
BigQuery: add missing query statistics.
shollyman May 10, 2018
626e5b2
Switch StatementType to a StringEnumType, more complete testing.
shollyman May 16, 2018
6a1e9f4
remove unused imports
shollyman May 16, 2018
ddb1ce4
push setEstimatedBytesProcessed in toPb()
shollyman May 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 212 additions & 0 deletions ...-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@

package com.google.cloud.bigquery;

import com.google.api.core.ApiFunction;
import com.google.api.services.bigquery.model.JobConfiguration;
import com.google.api.services.bigquery.model.JobStatistics2;
import com.google.api.services.bigquery.model.JobStatistics3;
import com.google.api.services.bigquery.model.JobStatistics4;
import com.google.cloud.StringEnumType;
import com.google.cloud.StringEnumValue;
import com.google.common.base.MoreObjects;
import com.google.common.base.MoreObjects.ToStringHelper;
import com.google.common.collect.Lists;
Expand Down Expand Up @@ -311,6 +314,7 @@ static LoadStatistics fromPb(com.google.api.services.bigquery.model.JobStatistic
}
}


/**
* A Google BigQuery Query Job statistics.
*/
Expand All @@ -320,18 +324,92 @@ public static class QueryStatistics extends JobStatistics {

private final Integer billingTier;
private final Boolean cacheHit;
private final String ddlOperationPerformed;
private final TableId ddlTargetTable;
private final Long estimatedBytesProcessed;
private final Long numDmlAffectedRows;
private final List<TableId> referencedTables;

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

private final StatementType statementType;
private final Long totalBytesBilled;
private final Long totalBytesProcessed;
private final Long totalPartitionsProcessed;
private final Long totalSlotMs;
private final List<QueryStage> queryPlan;
private final List<TimelineSample> timeline;
private final Schema schema;


/**
* StatementType represents possible types of SQL statements reported as part of the
* QueryStatistics of a BigQuery job.
*/
public static final class StatementType extends StringEnumValue {
private static final long serialVersionUID = 818920627219751204L;

private static final ApiFunction<String, StatementType> CONSTRUCTOR =
new ApiFunction<String, StatementType>() {
@Override
public StatementType apply(String constant) {
return new StatementType(constant);
}
};

private static final StringEnumType<StatementType> type = new StringEnumType(
StatementType.class,
CONSTRUCTOR);

public static final StatementType SELECT = type.createAndRegister("SELECT");
public static final StatementType UPDATE = type.createAndRegister("UPDATE");
public static final StatementType INSERT = type.createAndRegister("INSERT");
public static final StatementType DELETE = type.createAndRegister("DELETE");
public static final StatementType CREATE_TABLE = type.createAndRegister("CREATE_TABLE");
public static final StatementType CREATE_TABLE_AS_SELECT = type.createAndRegister("CREATE_TABLE_AS_SELECT");
public static final StatementType CREATE_VIEW = type.createAndRegister("CREATE_VIEW");
public static final StatementType DROP_TABLE = type.createAndRegister("DROP_TABLE");
public static final StatementType DROP_VIEW = type.createAndRegister("DROP_VIEW");
public static final StatementType MERGE = type.createAndRegister("MERGE");

private StatementType(String constant) {
super(constant);
}

/**
* Get the StatementType for the given String constant, and throw an exception if the constant is
* not recognized.
*/
public static StatementType valueOfStrict(String constant) {
return type.valueOfStrict(constant);
}

/**
* Get the State for the given String constant, and allow unrecognized values.
*/
public static StatementType valueOf(String constant) {
return type.valueOf(constant);
}

/**
* Return the known values for State.
*/
public static StatementType[] values() {
return type.values();
}
}

static final class Builder extends JobStatistics.Builder<QueryStatistics, Builder> {

private Integer billingTier;
private Boolean cacheHit;
private String ddlOperationPerformed;
private TableId ddlTargetTable;
private Long estimatedBytesProcessed;
private Long numDmlAffectedRows;
private List<TableId> referencedTables;
private StatementType statementType;
private Long totalBytesBilled;
private Long totalBytesProcessed;
private Long totalPartitionsProcessed;
private Long totalSlotMs;
private List<QueryStage> queryPlan;
private List<TimelineSample> timeline;
private Schema schema;
Expand All @@ -343,8 +421,25 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP
if (statisticsPb.getQuery() != null) {
this.billingTier = statisticsPb.getQuery().getBillingTier();
this.cacheHit = statisticsPb.getQuery().getCacheHit();
this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed();
if (statisticsPb.getQuery().getDdlTargetTable() != null) {
this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable());
}
this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed();
this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows();
this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled();
this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed();
this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed();
this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs();
if (statisticsPb.getQuery().getStatementType() != null) {
this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType());
}

if (statisticsPb.getQuery().getReferencedTables() != null) {
this.referencedTables =
Lists.transform(
statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION);
}
if (statisticsPb.getQuery().getQueryPlan() != null) {
this.queryPlan =
Lists.transform(
Expand All @@ -371,6 +466,41 @@ Builder setCacheHit(Boolean cacheHit) {
return self();
}

Builder setDDLOperationPerformed(String ddlOperationPerformed) {
this.ddlOperationPerformed = ddlOperationPerformed;
return self();
}

Builder setDDLTargetTable(TableId ddlTargetTable) {
this.ddlTargetTable = ddlTargetTable;
return self();
}

Builder setEstimatedBytesProcessed(Long estimatedBytesProcessed) {
this.estimatedBytesProcessed = estimatedBytesProcessed;
return self();
}

Builder setNumDmlAffectedRows(Long numDmlAffectedRows) {
this.numDmlAffectedRows = numDmlAffectedRows;
return self();
}

Builder setReferenceTables(List<TableId> referencedTables) {
this.referencedTables = referencedTables;
return self();
}

Builder setStatementType(StatementType statementType) {
this.statementType = statementType;
return self();
}

Builder setStatementType(String strStatementType) {
this.statementType = StatementType.valueOf(strStatementType);
return self();
}

Builder setTotalBytesBilled(Long totalBytesBilled) {
this.totalBytesBilled = totalBytesBilled;
return self();
Expand All @@ -381,6 +511,16 @@ Builder setTotalBytesProcessed(Long totalBytesProcessed) {
return self();
}

Builder setTotalPartitionsProcessed(Long totalPartitionsProcessed) {
this.totalPartitionsProcessed = totalPartitionsProcessed;
return self();
}

Builder setTotalSlotMs(Long totalSlotMs) {
this.totalSlotMs = totalSlotMs;
return self();
}

Builder setQueryPlan(List<QueryStage> queryPlan) {
this.queryPlan = queryPlan;
return self();
Expand All @@ -406,8 +546,16 @@ private QueryStatistics(Builder builder) {
super(builder);
this.billingTier = builder.billingTier;
this.cacheHit = builder.cacheHit;
this.ddlOperationPerformed = builder.ddlOperationPerformed;
this.ddlTargetTable = builder.ddlTargetTable;
this.estimatedBytesProcessed = builder.estimatedBytesProcessed;
this.numDmlAffectedRows = builder.numDmlAffectedRows;
this.referencedTables = builder.referencedTables;
this.statementType = builder.statementType;
this.totalBytesBilled = builder.totalBytesBilled;
this.totalBytesProcessed = builder.totalBytesProcessed;
this.totalPartitionsProcessed = builder.totalPartitionsProcessed;
this.totalSlotMs = builder.totalSlotMs;
this.queryPlan = builder.queryPlan;
this.timeline = builder.timeline;
this.schema = builder.schema;
Expand All @@ -432,6 +580,47 @@ public Boolean getCacheHit() {
return cacheHit;
}

/**
* [BETA] For DDL queries, returns the operation applied to the DDL target table.
*/
public String getDdlOperationPerformed() { return ddlOperationPerformed; }

/**
* [BETA] For DDL queries, returns the TableID of the targeted table.
*/
public TableId getDdlTargetTable() { return ddlTargetTable; }

/**
* The original estimate of bytes processed for the job.
*/
public Long getEstimatedBytesProcessed() { return estimatedBytesProcessed; }

/**
* The number of rows affected by a DML statement.
* Present only for DML statements INSERT, UPDATE or DELETE.
*/
public Long getNumDmlAffectedRows() { return numDmlAffectedRows; }

/**
* Referenced tables for the job.
* Queries that reference more than 50 tables will not have a complete list.
*/
public List<TableId> getReferencedTables() { return referencedTables; }

/**
* [BETA] The type of query statement, if valid.
* Possible values include:
* SELECT
* INSERT
* UPDATE
* DELETE
* CREATE_TABLE
* CREATE_TABLE_AS_SELECT
* DROP_TABLE
* CREATE_VIEW
* DROP_VIEW
*/
public StatementType getStatementType() { return statementType; }

/**
* Returns the total number of bytes billed for the job.
Expand All @@ -448,6 +637,15 @@ public Long getTotalBytesProcessed() {
return totalBytesProcessed;
}

/**
* Total number of partitions processed from all partitioned tables referenced in the job.
*/
public Long getTotalPartitionsProcessed() { return totalPartitionsProcessed; }

/**
* Returns the slot-milliseconds consumed by the query.
*/
public Long getTotalSlotMs() { return totalSlotMs; }

/**
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.
Expand Down Expand Up @@ -509,8 +707,22 @@ com.google.api.services.bigquery.model.JobStatistics toPb() {
JobStatistics2 queryStatisticsPb = new JobStatistics2();
queryStatisticsPb.setBillingTier(billingTier);
queryStatisticsPb.setCacheHit(cacheHit);
queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed);
queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed);
queryStatisticsPb.setTotalBytesBilled(totalBytesBilled);
queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed);
queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed);
queryStatisticsPb.setTotalSlotMs(totalSlotMs);
if (ddlTargetTable != null) {
queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb());
}

if (referencedTables != null) {
queryStatisticsPb.setReferencedTables(Lists.transform(referencedTables, TableId.TO_PB_FUNCTION));
}
if (statementType != null) {
queryStatisticsPb.setStatementType(statementType.toString());
}
if (queryPlan != null) {
queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION));
}
Expand Down
42 changes: 41 additions & 1 deletion ...ents/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,22 @@ public class JobStatisticsTest {

private static final Integer BILLING_TIER = 42;
private static final Boolean CACHE_HIT = true;
private static final String DDL_OPERATION_PERFORMED = "SKIP";
private static final TableId DDL_TARGET_TABLE = TableId.of("foo", "bar", "baz");
private static final Long ESTIMATE_BYTES_PROCESSED = 101L;
private static final Long NUM_DML_AFFECTED_ROWS = 88L;
private static final QueryStatistics.StatementType STATEMENT_TYPE = QueryStatistics.StatementType.SELECT;
private static final Long TOTAL_BYTES_BILLED = 24L;
private static final Long TOTAL_BYTES_PROCESSED = 42L;
private static final Long TOTAL_PARTITION_PROCESSED = 63L;
private static final Long TOTAL_SLOT_MS = 10202L;
private static final Long INPUT_BYTES = 1L;
private static final Long INPUT_FILES = 2L;
private static final Long OUTPUT_BYTES = 3L;
private static final Long OUTPUT_ROWS = 4L;
private static final List<TableId> REFERENCED_TABLES = ImmutableList.of(
TableId.of("foo", "bar", "table1"),
TableId.of("foo","bar","table2"));
private static final List<Long> FILE_COUNT = ImmutableList.of(1L, 2L, 3L);
private static final Long CREATION_TIME = 10L;
private static final Long END_TIME = 20L;
Expand Down Expand Up @@ -112,8 +122,16 @@ public class JobStatisticsTest {
.setStartTime(START_TIME)
.setBillingTier(BILLING_TIER)
.setCacheHit(CACHE_HIT)
.setDDLOperationPerformed(DDL_OPERATION_PERFORMED)
.setDDLTargetTable(DDL_TARGET_TABLE)
.setEstimatedBytesProcessed(ESTIMATE_BYTES_PROCESSED)
.setNumDmlAffectedRows(NUM_DML_AFFECTED_ROWS)
.setReferenceTables(REFERENCED_TABLES)
.setStatementType(STATEMENT_TYPE)
.setTotalBytesBilled(TOTAL_BYTES_BILLED)
.setTotalBytesProcessed(TOTAL_BYTES_PROCESSED)
.setTotalPartitionsProcessed(TOTAL_PARTITION_PROCESSED)
.setTotalSlotMs(TOTAL_SLOT_MS)
.setQueryPlan(QUERY_PLAN)
.setTimeline(TIMELINE)
.setSchema(SCHEMA)
Expand Down Expand Up @@ -146,9 +164,16 @@ public void testBuilder() {
assertEquals(END_TIME, QUERY_STATISTICS.getEndTime());
assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier());
assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit());
assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed());
assertEquals(DDL_TARGET_TABLE, QUERY_STATISTICS.getDdlTargetTable());
assertEquals(ESTIMATE_BYTES_PROCESSED, QUERY_STATISTICS.getEstimatedBytesProcessed());
assertEquals(NUM_DML_AFFECTED_ROWS, QUERY_STATISTICS.getNumDmlAffectedRows());
assertEquals(REFERENCED_TABLES, QUERY_STATISTICS.getReferencedTables());
assertEquals(STATEMENT_TYPE, QUERY_STATISTICS.getStatementType());
assertEquals(TOTAL_BYTES_BILLED, QUERY_STATISTICS.getTotalBytesBilled());
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed());
assertEquals(TOTAL_PARTITION_PROCESSED, QUERY_STATISTICS.getTotalPartitionsProcessed());
assertEquals(TOTAL_SLOT_MS, QUERY_STATISTICS.getTotalSlotMs());
assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan());
assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline());

Expand All @@ -165,8 +190,15 @@ public void testBuilder() {
assertEquals(END_TIME, QUERY_STATISTICS_INCOMPLETE.getEndTime());
assertEquals(BILLING_TIER, QUERY_STATISTICS_INCOMPLETE.getBillingTier());
assertEquals(CACHE_HIT, QUERY_STATISTICS_INCOMPLETE.getCacheHit());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlOperationPerformed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getDdlTargetTable());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getEstimatedBytesProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getNumDmlAffectedRows());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesBilled());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalBytesProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalPartitionsProcessed());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getTotalSlotMs());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getReferencedTables());
assertEquals(null, QUERY_STATISTICS_INCOMPLETE.getQueryPlan());
}

Expand Down Expand Up @@ -235,10 +267,18 @@ private void compareQueryStatistics(QueryStatistics expected, QueryStatistics va
compareStatistics(expected, value);
assertEquals(expected.getBillingTier(), value.getBillingTier());
assertEquals(expected.getCacheHit(), value.getCacheHit());
assertEquals(expected.getDdlOperationPerformed(), value.getDdlOperationPerformed());
assertEquals(expected.getDdlTargetTable(), value.getDdlTargetTable());
assertEquals(expected.getEstimatedBytesProcessed(), value.getEstimatedBytesProcessed());
assertEquals(expected.getTotalBytesBilled(), value.getTotalBytesBilled());
assertEquals(expected.getTotalBytesProcessed(), value.getTotalBytesProcessed());
assertEquals(expected.getTotalPartitionsProcessed(), value.getTotalPartitionsProcessed());
assertEquals(expected.getTotalSlotMs(), value.getTotalSlotMs());
assertEquals(expected.getQueryPlan(), value.getQueryPlan());
assertEquals(expected.getReferencedTables(), value.getReferencedTables());
assertEquals(expected.getSchema(), value.getSchema());
assertEquals(expected.getStatementType(), value.getStatementType());
assertEquals(expected.getTimeline(), value.getTimeline());
}

private void compareStatistics(JobStatistics expected, JobStatistics value) {
Expand Down