From 1f808374626c1d50e37040c2a7bf3fe72468b883 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Thu, 13 Mar 2025 13:25:22 -0400 Subject: [PATCH 1/3] Use Schema builder APIs compatible with Avro 1.8 in Storage Write API translator --- .../gcp/bigquery/AvroGenericRecordToStorageApiProto.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java index 3813b000a65b..e31303a9bc55 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java @@ -350,10 +350,11 @@ private static TableFieldSchema fieldDescriptorFromAvroField(org.apache.avro.Sch throw new RuntimeException("Unexpected null element type!"); } TableFieldSchema keyFieldSchema = - fieldDescriptorFromAvroField(new Schema.Field("key", keyType, "key of the map entry")); + fieldDescriptorFromAvroField( + new Schema.Field("key", keyType, "key of the map entry", null)); TableFieldSchema valueFieldSchema = fieldDescriptorFromAvroField( - new Schema.Field("value", valueType, "value of the map entry")); + new Schema.Field("value", valueType, "value of the map entry", null)); builder = builder .setType(TableFieldSchema.Type.STRUCT) @@ -371,7 +372,8 @@ private static TableFieldSchema fieldDescriptorFromAvroField(org.apache.avro.Sch elementType.getType() != Schema.Type.UNION, "Multiple non-null union types are not supported."); TableFieldSchema unionFieldSchema = - fieldDescriptorFromAvroField(new Schema.Field(field.name(), elementType, field.doc())); + fieldDescriptorFromAvroField( + new Schema.Field(field.name(), elementType, field.doc(), null)); builder = builder .setType(unionFieldSchema.getType()) From 94401f1f0064a795ab4ef7253be0bdd593faff4d Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Thu, 13 Mar 2025 13:58:25 -0400 Subject: [PATCH 2/3] Suppress nullness check --- .../sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java index e31303a9bc55..fd9dffc260e7 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java @@ -311,6 +311,7 @@ public static DynamicMessage messageFromGenericRecord( return builder.build(); } + @SuppressWarnings("nullness") private static TableFieldSchema fieldDescriptorFromAvroField(org.apache.avro.Schema.Field field) { @Nullable Schema schema = field.schema(); Preconditions.checkNotNull(schema, "Unexpected null schema!"); From 135d1295b0f9a5e1d64162895a8cb5b14bace672 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Thu, 13 Mar 2025 16:42:03 -0400 Subject: [PATCH 3/3] Update CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index c75a42d25945..b7b86b2313e9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -88,6 +88,7 @@ * (Python) Fixed occasional pipeline stuckness that was affecting Python 3.11 users ([#33966](https://github.com/apache/beam/issues/33966)). * (Java) Fixed TIME field encodings for BigQuery Storage API writes on GenericRecords ([#34059](https://github.com/apache/beam/pull/34059)). * (Java) Fixed a race condition in JdbcIO which could cause hangs trying to acquire a connection ([#34058](https://github.com/apache/beam/pull/34058)). +* (Java) Fix BigQuery Storage Write compatibility with Avro 1.8 ([#34281](https://github.com/apache/beam/pull/34281)). ## Security Fixes * Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).