From 007c3fb425b7d9038c25659158259562d9e2817a Mon Sep 17 00:00:00 2001 From: Mike Kaplinskiy Date: Sun, 15 Apr 2018 14:56:14 -0700 Subject: [PATCH 1/2] Add support for new api features in BigQuery TimePartitioning. The two features are requirePartitionFilter & field. Also this moves to using a builder instead of using more `of` methods since the number of combinations is large. --- .../cloud/bigquery/TimePartitioning.java | 132 ++++++++++++++++-- .../cloud/bigquery/TimePartitioningTest.java | 25 +++- 2 files changed, 148 insertions(+), 9 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java index 3359736ca1a0..bcc8caf6d498 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.core.BetaApi; import com.google.common.base.MoreObjects; import java.io.Serializable; @@ -36,6 +37,8 @@ public final class TimePartitioning implements Serializable { private final Type type; private final Long expirationMs; + private final Boolean requirePartitionFilter; + private final String field; /** * The type of time partitioning. Currently, the only type supported is {@code DAY}, which will @@ -49,9 +52,92 @@ public enum Type { DAY } - private TimePartitioning(Type type, Long expirationMs) { - this.type = checkNotNull(type); - this.expirationMs = expirationMs; + /** + * A builder for {@code TimePartitioning} objects. + */ + public abstract static class Builder { + abstract Builder setType(Type type); + + /** + * Sets the number of milliseconds for which to keep the storage for a partition. When expired, + * the storage for the partition is reclaimed. + */ + public abstract Builder setExpirationMs(long expirationMs); + + /** + * Sets whether queries over this table require a partition where clause. + */ + @BetaApi + public abstract Builder setRequirePartitionFilter(boolean requirePartitionFilter); + + /** + * Sets the field to partition the table by. If not set, the table is partitioned by pseudo + * column '_PARTITIONTIME'. + */ + @BetaApi + public abstract Builder setField(String field); + + public abstract TimePartitioning build(); + } + + static class BuilderImpl extends Builder { + + private Type type; + private Long expirationMs; + private Boolean requirePartitionFilter; + private String field; + + BuilderImpl() {} + + BuilderImpl(TimePartitioning tp) { + this.type = tp.getType(); + this.expirationMs = tp.getExpirationMs(); + this.requirePartitionFilter = tp.getRequirePartitionFilter(); + this.field = tp.getField(); + } + + BuilderImpl(com.google.api.services.bigquery.model.TimePartitioning tp) { + this.type = Type.valueOf(tp.getType()); + this.expirationMs = tp.getExpirationMs(); + this.requirePartitionFilter = tp.getRequirePartitionFilter(); + this.field = tp.getField(); + } + + @Override + Builder setType(Type type) { + this.type = type; + return this; + } + + @Override + public Builder setExpirationMs(long expirationMs) { + this.expirationMs = expirationMs; + return this; + } + + @Override + public Builder setRequirePartitionFilter(boolean requirePartitionFilter) { + this.requirePartitionFilter = requirePartitionFilter; + return this; + } + + @Override + public Builder setField(String field) { + this.field = field; + return this; + } + + @Override + public TimePartitioning build() { + return new TimePartitioning(this); + } + } + + private TimePartitioning(BuilderImpl builder) { + this.type = checkNotNull(builder.type); + this.expirationMs = builder.expirationMs; + this.requirePartitionFilter = builder.requirePartitionFilter; + this.field = builder.field; } @@ -72,17 +158,37 @@ public Long getExpirationMs() { return expirationMs; } + + /** + * If not set, the table is partitioned by pseudo column '_PARTITIONTIME'; if set, the table is + * partitioned by this field. + */ + public String getField() { + return field; + } + + + /** + * If set to true, queries over this table require a partition filter (that can be used for + * partition elimination) to be specified. + */ + public Boolean getRequirePartitionFilter() { + return requirePartitionFilter; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("type", type) .add("expirationMs", expirationMs) + .add("requirePartitionFilter", requirePartitionFilter) + .add("field", field) .toString(); } @Override public int hashCode() { - return Objects.hash(type, expirationMs); + return Objects.hash(type, expirationMs, requirePartitionFilter, field); } @Override @@ -92,13 +198,22 @@ public boolean equals(Object obj) { && Objects.equals(toPb(), ((TimePartitioning) obj).toPb()); } + /** + * Returns a {@code TimePartitioning} object given the time partitioning type. Currently, the only + * type supported is {@link Type#DAY}, which will generate one partition per day based on data + * loading time. + */ + public static Builder newBuilder(Type type) { + return new BuilderImpl().setType(type); + } + /** * Returns a {@code TimePartitioning} object given the time partitioning type. Currently, the only * type supported is {@link Type#DAY}, which will generate one partition per day based on data * loading time. */ public static TimePartitioning of(Type type) { - return new TimePartitioning(type, null); + return newBuilder(type).build(); } /** @@ -110,7 +225,7 @@ public static TimePartitioning of(Type type) { * @param expirationMs the number of milliseconds for which to keep the storage for a partition */ public static TimePartitioning of(Type type, long expirationMs) { - return new TimePartitioning(type, expirationMs); + return newBuilder(type).setExpirationMs(expirationMs).build(); } com.google.api.services.bigquery.model.TimePartitioning toPb() { @@ -118,12 +233,13 @@ com.google.api.services.bigquery.model.TimePartitioning toPb() { new com.google.api.services.bigquery.model.TimePartitioning(); partitioningPb.setType(type.name()); partitioningPb.setExpirationMs(expirationMs); + partitioningPb.setRequirePartitionFilter(requirePartitionFilter); + partitioningPb.setField(field); return partitioningPb; } static TimePartitioning fromPb( com.google.api.services.bigquery.model.TimePartitioning partitioningPb) { - return new TimePartitioning( - Type.valueOf(partitioningPb.getType()), partitioningPb.getExpirationMs()); + return new BuilderImpl(partitioningPb).build(); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java index 037ca4625419..9d88c9c74f42 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java @@ -29,8 +29,14 @@ public class TimePartitioningTest { private static final Type TYPE = Type.DAY; private static final long EXPIRATION_MS = 42; + private static final boolean REQUIRE_PARTITION_FILTER = false; + private static final String FIELD = "field"; private static final TimePartitioning TIME_PARTITIONING = - TimePartitioning.of(TYPE, EXPIRATION_MS); + TimePartitioning.newBuilder(TYPE) + .setExpirationMs(EXPIRATION_MS) + .setRequirePartitionFilter(REQUIRE_PARTITION_FILTER) + .setField(FIELD) + .build(); @Rule public ExpectedException thrown = ExpectedException.none(); @@ -39,11 +45,26 @@ public class TimePartitioningTest { public void testOf() { assertEquals(TYPE, TIME_PARTITIONING.getType()); assertEquals(EXPIRATION_MS, TIME_PARTITIONING.getExpirationMs().longValue()); + assertEquals(REQUIRE_PARTITION_FILTER, TIME_PARTITIONING.getRequirePartitionFilter()); + assertEquals(FIELD, TIME_PARTITIONING.getField()); TimePartitioning partitioning = TimePartitioning.of(TYPE); assertEquals(TYPE, partitioning.getType()); assertNull(partitioning.getExpirationMs()); } + @Test + public void testBuilder() { + TimePartitioning partitioning = TimePartitioning.newBuilder(TYPE).build(); + assertEquals(TYPE, partitioning.getType()); + assertNull(partitioning.getExpirationMs()); + assertNull(partitioning.getRequirePartitionFilter()); + assertNull(partitioning.getField()); + partitioning = TimePartitioning.newBuilder(TYPE).setExpirationMs(100).build(); + assertEquals(TYPE, partitioning.getType()); + assertEquals(100, (long) partitioning.getExpirationMs()); + assertNull(partitioning.getRequirePartitionFilter()); + assertNull(partitioning.getField()); + } @Test public void testTypeOf_Npe() { @@ -68,6 +89,8 @@ private void compareTimePartitioning(TimePartitioning expected, TimePartitioning assertEquals(expected, value); assertEquals(expected.getType(), value.getType()); assertEquals(expected.getExpirationMs(), value.getExpirationMs()); + assertEquals(expected.getRequirePartitionFilter(), value.getRequirePartitionFilter()); + assertEquals(expected.getField(), value.getField()); assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); } From 35a0287c1cf802eafef3861da1a86e5fd2864410 Mon Sep 17 00:00:00 2001 From: Mike Kaplinskiy Date: Mon, 16 Apr 2018 11:36:09 -0700 Subject: [PATCH 2/2] Move TimePartitioning to @AutoValue --- .../cloud/bigquery/TimePartitioning.java | 167 ++++-------------- .../cloud/bigquery/TimePartitioningTest.java | 2 +- 2 files changed, 39 insertions(+), 130 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java index bcc8caf6d498..659c902b56b8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TimePartitioning.java @@ -19,8 +19,10 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.core.BetaApi; +import com.google.auto.value.AutoValue; import com.google.common.base.MoreObjects; +import javax.annotation.Nullable; import java.io.Serializable; import java.util.Objects; @@ -31,15 +33,11 @@ * * @see Partitioned Tables */ -public final class TimePartitioning implements Serializable { +@AutoValue +public abstract class TimePartitioning implements Serializable { private static final long serialVersionUID = -8565064035346940951L; - private final Type type; - private final Long expirationMs; - private final Boolean requirePartitionFilter; - private final String field; - /** * The type of time partitioning. Currently, the only type supported is {@code DAY}, which will * generate one partition per day based on data loading time. @@ -52,150 +50,57 @@ public enum Type { DAY } - /** - * A builder for {@code TimePartitioning} objects. - */ - public abstract static class Builder { - abstract Builder setType(Type type); - - /** - * Sets the number of milliseconds for which to keep the storage for a partition. When expired, - * the storage for the partition is reclaimed. - */ - public abstract Builder setExpirationMs(long expirationMs); - - /** - * Sets whether queries over this table require a partition where clause. - */ - @BetaApi - public abstract Builder setRequirePartitionFilter(boolean requirePartitionFilter); - - /** - * Sets the field to partition the table by. If not set, the table is partitioned by pseudo - * column '_PARTITIONTIME'. - */ - @BetaApi - public abstract Builder setField(String field); - - public abstract TimePartitioning build(); + TimePartitioning() { + // Users cannot extend this, but AutoValue can. } - static class BuilderImpl extends Builder { - - private Type type; - private Long expirationMs; - private Boolean requirePartitionFilter; - private String field; - - BuilderImpl() {} - - BuilderImpl(TimePartitioning tp) { - this.type = tp.getType(); - this.expirationMs = tp.getExpirationMs(); - this.requirePartitionFilter = tp.getRequirePartitionFilter(); - this.field = tp.getField(); - } - - BuilderImpl(com.google.api.services.bigquery.model.TimePartitioning tp) { - this.type = Type.valueOf(tp.getType()); - this.expirationMs = tp.getExpirationMs(); - this.requirePartitionFilter = tp.getRequirePartitionFilter(); - this.field = tp.getField(); - } - - @Override - Builder setType(Type type) { - this.type = type; - return this; - } - - @Override - public Builder setExpirationMs(long expirationMs) { - this.expirationMs = expirationMs; - return this; - } - - @Override - public Builder setRequirePartitionFilter(boolean requirePartitionFilter) { - this.requirePartitionFilter = requirePartitionFilter; - return this; - } - - @Override - public Builder setField(String field) { - this.field = field; - return this; - } - - @Override - public TimePartitioning build() { - return new TimePartitioning(this); - } - } - - private TimePartitioning(BuilderImpl builder) { - this.type = checkNotNull(builder.type); - this.expirationMs = builder.expirationMs; - this.requirePartitionFilter = builder.requirePartitionFilter; - this.field = builder.field; - } - - /** * Returns the time partitioning type. Currently, the only type supported is {@link Type#DAY}, * which will generate one partition per day based on data loading time. */ - public Type getType() { - return type; - } + public abstract Type getType(); /** * Returns the number of milliseconds for which to keep the storage for a partition. When expired, * the storage for the partition is reclaimed. */ - public Long getExpirationMs() { - return expirationMs; - } + @Nullable + public abstract Long getExpirationMs(); /** * If not set, the table is partitioned by pseudo column '_PARTITIONTIME'; if set, the table is * partitioned by this field. */ - public String getField() { - return field; - } + @BetaApi + @Nullable + public abstract String getField(); /** * If set to true, queries over this table require a partition filter (that can be used for * partition elimination) to be specified. */ - public Boolean getRequirePartitionFilter() { - return requirePartitionFilter; - } + @BetaApi + @Nullable + public abstract Boolean getRequirePartitionFilter(); - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("type", type) - .add("expirationMs", expirationMs) - .add("requirePartitionFilter", requirePartitionFilter) - .add("field", field) - .toString(); - } + public abstract Builder toBuilder(); - @Override - public int hashCode() { - return Objects.hash(type, expirationMs, requirePartitionFilter, field); - } + @AutoValue.Builder + public abstract static class Builder { + abstract Builder setType(Type type); + + public abstract Builder setExpirationMs(Long expirationMs); + + @BetaApi + public abstract Builder setRequirePartitionFilter(Boolean requirePartitionFilter); - @Override - public boolean equals(Object obj) { - return obj == this - || obj instanceof TimePartitioning - && Objects.equals(toPb(), ((TimePartitioning) obj).toPb()); + @BetaApi + public abstract Builder setField(String field); + + public abstract TimePartitioning build(); } /** @@ -204,7 +109,7 @@ public boolean equals(Object obj) { * loading time. */ public static Builder newBuilder(Type type) { - return new BuilderImpl().setType(type); + return new AutoValue_TimePartitioning.Builder().setType(type); } /** @@ -231,15 +136,19 @@ public static TimePartitioning of(Type type, long expirationMs) { com.google.api.services.bigquery.model.TimePartitioning toPb() { com.google.api.services.bigquery.model.TimePartitioning partitioningPb = new com.google.api.services.bigquery.model.TimePartitioning(); - partitioningPb.setType(type.name()); - partitioningPb.setExpirationMs(expirationMs); - partitioningPb.setRequirePartitionFilter(requirePartitionFilter); - partitioningPb.setField(field); + partitioningPb.setType(getType().name()); + partitioningPb.setExpirationMs(getExpirationMs()); + partitioningPb.setRequirePartitionFilter(getRequirePartitionFilter()); + partitioningPb.setField(getField()); return partitioningPb; } static TimePartitioning fromPb( com.google.api.services.bigquery.model.TimePartitioning partitioningPb) { - return new BuilderImpl(partitioningPb).build(); + return newBuilder(Type.valueOf(partitioningPb.getType())) + .setExpirationMs(partitioningPb.getExpirationMs()) + .setField(partitioningPb.getField()) + .setRequirePartitionFilter(partitioningPb.getRequirePartitionFilter()) + .build(); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java index 9d88c9c74f42..75488afceb27 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/TimePartitioningTest.java @@ -59,7 +59,7 @@ public void testBuilder() { assertNull(partitioning.getExpirationMs()); assertNull(partitioning.getRequirePartitionFilter()); assertNull(partitioning.getField()); - partitioning = TimePartitioning.newBuilder(TYPE).setExpirationMs(100).build(); + partitioning = TimePartitioning.newBuilder(TYPE).setExpirationMs(100L).build(); assertEquals(TYPE, partitioning.getType()); assertEquals(100, (long) partitioning.getExpirationMs()); assertNull(partitioning.getRequirePartitionFilter());