From b64287c66f39daeb6dfa0eff6f8e0158ad3c17ad Mon Sep 17 00:00:00 2001 From: mburuian6 Date: Fri, 5 Sep 2025 17:15:58 +0200 Subject: [PATCH 1/2] Logs the BigQuery error that caused the load job failure (resolves #18419) --- .../sdk/io/gcp/bigquery/BigQueryHelpers.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java index 129c8314fc80..7f49b28dd4df 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java @@ -25,6 +25,7 @@ import com.google.api.client.util.Sleeper; import com.google.api.services.bigquery.model.Clustering; import com.google.api.services.bigquery.model.Dataset; +import com.google.api.services.bigquery.model.ErrorProto; import com.google.api.services.bigquery.model.Job; import com.google.api.services.bigquery.model.JobReference; import com.google.api.services.bigquery.model.JobStatus; @@ -205,6 +206,7 @@ static class PendingJob implements Serializable { void runJob() throws IOException { ++currentAttempt; if (!shouldRetry()) { + logBigQueryError(lastJobAttempted); throw new RuntimeException( String.format( "Failed to create job with prefix %s, " @@ -281,6 +283,21 @@ boolean pollJob() throws IOException { boolean shouldRetry() { return currentAttempt < maxRetries + 1; } + + void logBigQueryError(@Nullable Job job) { + if (job == null || !parseStatus(job).equals(Status.FAILED)) { + return; + } + + List jobErrors = job.getStatus().getErrors(); + ErrorProto finalError = job.getStatus().getErrorResult(); + ErrorProto causativeError = jobErrors.get(jobErrors.size() - 1); + + LOG.error( + String.format( + "BigQuery Error : %s due to : %s", + finalError.getMessage(), causativeError.getMessage())); + } } static class RetryJobId implements Serializable { From 43bd1d031e4308f4491711a00986ee305c9f7965 Mon Sep 17 00:00:00 2001 From: mburuian6 Date: Fri, 5 Sep 2025 21:19:22 +0200 Subject: [PATCH 2/2] Checks that the BigQuery root cause message to be logged is not empty (resolves #18419) --- .../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java index 7f49b28dd4df..d468ffbea43c 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java @@ -290,13 +290,13 @@ void logBigQueryError(@Nullable Job job) { } List jobErrors = job.getStatus().getErrors(); - ErrorProto finalError = job.getStatus().getErrorResult(); - ErrorProto causativeError = jobErrors.get(jobErrors.size() - 1); + String finalError = job.getStatus().getErrorResult().getMessage(); + String causativeError = + jobErrors != null && !jobErrors.isEmpty() + ? String.format(" due to: %s", jobErrors.get(jobErrors.size() - 1).getMessage()) + : ""; - LOG.error( - String.format( - "BigQuery Error : %s due to : %s", - finalError.getMessage(), causativeError.getMessage())); + LOG.error(String.format("BigQuery Error : %s %s", finalError, causativeError)); } }