From 9aaba5bd46f8cd9f6c13c338298150ceaec61888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 11 Dec 2024 12:02:59 +0800 Subject: [PATCH 1/2] [core] Check file size after write bundle of records --- .../main/java/org/apache/paimon/io/RollingFileWriter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java index 109b7574304e..63a829324253 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java @@ -65,9 +65,9 @@ public long targetFileSize() { } @VisibleForTesting - boolean rollingFile() throws IOException { + boolean rollingFile(boolean forceCheck) throws IOException { return currentWriter.reachTargetSize( - recordCount % CHECK_ROLLING_RECORD_CNT == 0, targetFileSize); + forceCheck || recordCount % CHECK_ROLLING_RECORD_CNT == 0, targetFileSize); } @Override @@ -81,7 +81,7 @@ public void write(T row) throws IOException { currentWriter.write(row); recordCount += 1; - if (rollingFile()) { + if (rollingFile(false)) { closeCurrentWriter(); } } catch (Throwable e) { @@ -105,7 +105,7 @@ public void writeBundle(BundleRecords bundle) throws IOException { currentWriter.writeBundle(bundle); recordCount += bundle.rowCount(); - if (rollingFile()) { + if (rollingFile(true)) { closeCurrentWriter(); } } catch (Throwable e) { From 3d201f0e951039968317e6c0237f9ac2bf7e1b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 11 Dec 2024 15:04:08 +0800 Subject: [PATCH 2/2] fix comment --- .../src/main/java/org/apache/paimon/io/RollingFileWriter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java index 63a829324253..29b9223b9a37 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java @@ -64,8 +64,7 @@ public long targetFileSize() { return targetFileSize; } - @VisibleForTesting - boolean rollingFile(boolean forceCheck) throws IOException { + private boolean rollingFile(boolean forceCheck) throws IOException { return currentWriter.reachTargetSize( forceCheck || recordCount % CHECK_ROLLING_RECORD_CNT == 0, targetFileSize); }