diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index 05e0856e5235..175e5f47f1fb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -323,6 +323,7 @@ public static class QueryStatistics extends JobStatistics { private final Long totalBytesBilled; private final Long totalBytesProcessed; private final List queryPlan; + private final List timeline; private final Schema schema; static final class Builder extends JobStatistics.Builder { @@ -332,6 +333,7 @@ static final class Builder extends JobStatistics.Builder queryPlan; + private List timeline; private Schema schema; private Builder() {} @@ -348,6 +350,11 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP Lists.transform( statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); } + if (statisticsPb.getQuery().getTimeline() != null) { + this.timeline = + Lists.transform( + statisticsPb.getQuery().getTimeline(), TimelineSample.FROM_PB_FUNCTION); + } if (statisticsPb.getQuery().getSchema() != null) { this.schema = Schema.fromPb(statisticsPb.getQuery().getSchema()); } @@ -379,6 +386,11 @@ Builder setQueryPlan(List queryPlan) { return self(); } + Builder setTimeline(List timeline) { + this.timeline = timeline; + return self(); + } + Builder setSchema(Schema schema) { this.schema = schema; return self(); @@ -397,6 +409,7 @@ private QueryStatistics(Builder builder) { this.totalBytesBilled = builder.totalBytesBilled; this.totalBytesProcessed = builder.totalBytesProcessed; this.queryPlan = builder.queryPlan; + this.timeline = builder.timeline; this.schema = builder.schema; } @@ -448,6 +461,14 @@ public List getQueryPlan() { return queryPlan; } + /** + * Return the timeline for the query, as a list of timeline samples. Each sample provides + * information about the overall progress of the query. Information includes time of the + * sample, progress reporting on active, completed, and + * pending units of work, as well as the cumulative estimation of slot-milliseconds consumed + * by the query. + */ + public List getTimeline() { return timeline; } /** * Returns the schema for the query result. Present only for successful dry run of @@ -465,6 +486,7 @@ ToStringHelper toStringHelper() { .add("totalBytesBilled", totalBytesBilled) .add("totalBytesProcessed", totalBytesProcessed) .add("queryPlan", queryPlan) + .add("timeline", timeline) .add("schema", schema); } @@ -492,6 +514,9 @@ com.google.api.services.bigquery.model.JobStatistics toPb() { if (queryPlan != null) { queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); } + if (timeline != null) { + queryStatisticsPb.setTimeline(Lists.transform(timeline, TimelineSample.TO_PB_FUNCTION)); + } if (schema != null) { queryStatisticsPb.setSchema(schema.toPb()); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimelineSample.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimelineSample.java new file mode 100644 index 000000000000..ddc7cf289f32 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimelineSample.java @@ -0,0 +1,124 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.QueryTimelineSample; +import com.google.common.base.Function; +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; + +/** + * A specific timeline sample. This instruments work progress at a given point in time, providing + * information about work units active/pending/completed as well as cumulative slot-milliseconds. + */ +@AutoValue +public abstract class TimelineSample { + + private static final long serialVersionUID = 1L; + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setElapsedMs(Long elapsedMs); + + public abstract Builder setActiveUnits(Long activeUnits); + + public abstract Builder setCompletedUnits(Long completedUnits); + + public abstract Builder setPendingUnits(Long pendingUnits); + + public abstract Builder setSlotMillis(Long slotMillis); + + public abstract TimelineSample build(); + + } + + /** + * Returns the sample time as milliseconds elapsed since the start of query execution. + */ + @Nullable + public abstract Long getElapsedMs(); + + /** + * Returns the total number of work units currently being processed. + */ + @Nullable + public abstract Long getActiveUnits(); + + /** + * Returns the total number of work units completed by this query. + */ + @Nullable + public abstract Long getCompletedUnits(); + + /** + * Returns the number of work units remaining for the currently active stages. + */ + @Nullable + public abstract Long getPendingUnits(); + + /** + * Returns the cumulative slot-milliseconds consumed by the query. + */ + @Nullable + public abstract Long getSlotMillis(); + + /** + * return a builder for the {@TimelineSample} object. + */ + public abstract Builder toBuilder(); + + static Builder newBuilder() { + return new AutoValue_TimelineSample.Builder(); + } + + static TimelineSample fromPb(QueryTimelineSample sample) { + Builder builder = newBuilder(); + builder.setElapsedMs(sample.getElapsedMs()); + builder.setActiveUnits(sample.getActiveUnits()); + builder.setCompletedUnits(sample.getCompletedUnits()); + builder.setPendingUnits(sample.getPendingUnits()); + builder.setSlotMillis(sample.getTotalSlotMs()); + return builder.build(); + } + + QueryTimelineSample toPb() { + QueryTimelineSample sample = new QueryTimelineSample() + .setElapsedMs(getElapsedMs()) + .setActiveUnits(getActiveUnits()) + .setCompletedUnits(getCompletedUnits()) + .setPendingUnits(getPendingUnits()) + .setTotalSlotMs(getSlotMillis()); + return sample; + } + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public TimelineSample apply(QueryTimelineSample pb) { + return TimelineSample.fromPb(pb); + } + }; + + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public QueryTimelineSample apply(TimelineSample sample) { + return sample.toPb(); + } + }; + +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java index 573318b17cb5..be738c89b355 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java @@ -88,6 +88,22 @@ public class JobStatisticsTest { .setWriteRatioAvg(9.9) .setWriteRatioMax(10.10) .build(); + + private static final TimelineSample TIMELINE_SAMPLE1 = TimelineSample.newBuilder() + .setElapsedMs(1001L) + .setActiveUnits(100L) + .setCompletedUnits(200L) + .setPendingUnits(50L) + .setSlotMillis(12345L) + .build(); + private static final TimelineSample TIMELINE_SAMPLE2 = TimelineSample.newBuilder() + .setElapsedMs(2002L) + .setActiveUnits(48L) + .setCompletedUnits(302L) + .setPendingUnits(0L) + .setSlotMillis(23456L) + .build(); + private static final List TIMELINE = ImmutableList.of(TIMELINE_SAMPLE1, TIMELINE_SAMPLE2); private static final List QUERY_PLAN = ImmutableList.of(QUERY_STAGE); private static final Schema SCHEMA = Schema.of(Field.of("column", LegacySQLTypeName.DATETIME)); private static final QueryStatistics QUERY_STATISTICS = QueryStatistics.newBuilder() @@ -99,6 +115,7 @@ public class JobStatisticsTest { .setTotalBytesBilled(TOTAL_BYTES_BILLED) .setTotalBytesProcessed(TOTAL_BYTES_PROCESSED) .setQueryPlan(QUERY_PLAN) + .setTimeline(TIMELINE) .setSchema(SCHEMA) .build(); private static final QueryStatistics QUERY_STATISTICS_INCOMPLETE = QueryStatistics.newBuilder() @@ -133,6 +150,7 @@ public void testBuilder() { assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed()); assertEquals(TOTAL_BYTES_PROCESSED, QUERY_STATISTICS.getTotalBytesProcessed()); assertEquals(QUERY_PLAN, QUERY_STATISTICS.getQueryPlan()); + assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline()); assertEquals(CREATION_TIME, LOAD_STATISTICS_INCOMPLETE.getCreationTime()); assertEquals(START_TIME, LOAD_STATISTICS_INCOMPLETE.getStartTime()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimelineSampleTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimelineSampleTest.java new file mode 100644 index 000000000000..8cf610ef45cd --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimelineSampleTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class TimelineSampleTest { + + private static final Long ELAPSED_MS = 1001L; + private static final Long ACTIVE_UNITS = 500L; + private static final Long COMPLETED_UNITS = 843L; + private static final Long PENDING_UNITS = 14L; + private static final Long SLOT_MILLIS = 1220012L; + + private static final TimelineSample TIMELINE_SAMPLE = TimelineSample.newBuilder() + .setElapsedMs(ELAPSED_MS) + .setActiveUnits(ACTIVE_UNITS) + .setCompletedUnits(COMPLETED_UNITS) + .setPendingUnits(PENDING_UNITS) + .setSlotMillis(SLOT_MILLIS) + .build(); + + + + @Test + public void testTimelineSampleBuilder() { + assertEquals(ELAPSED_MS, TIMELINE_SAMPLE.getElapsedMs()); + assertEquals(ACTIVE_UNITS, TIMELINE_SAMPLE.getActiveUnits()); + assertEquals(COMPLETED_UNITS, TIMELINE_SAMPLE.getCompletedUnits()); + assertEquals(PENDING_UNITS, TIMELINE_SAMPLE.getPendingUnits()); + assertEquals(SLOT_MILLIS, TIMELINE_SAMPLE.getSlotMillis()); + } + + @Test + public void TestEquals() { + assertEquals(TIMELINE_SAMPLE, TIMELINE_SAMPLE); + } + +} diff --git a/pom.xml b/pom.xml index 4379b2b7dcef..e909d6540de2 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ com.google.apis google-api-services-bigquery - v2-rev377-1.23.0 + v2-rev383-1.23.0 com.google.apis