From 0eb3763a761297cc09bbee81fd094d179d9c59f2 Mon Sep 17 00:00:00 2001 From: yangjiang Date: Wed, 11 Aug 2021 15:54:41 +0800 Subject: [PATCH] [PARQUET-2073] Fix estimate remaining row count in ColumnWriteStoreBase. --- .../org/apache/parquet/column/impl/ColumnWriteStoreBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java index a5e7836b3a..8cfdace8de 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java @@ -237,10 +237,11 @@ private void sizeCheck() { } else { rowCountForNextRowCountCheck = min(rowCountForNextRowCountCheck, writer.getRowsWrittenSoFar() + pageRowCountLimit); } + //estimate remaining row count by previous input for next row count check long rowsToFillPage = usedMem == 0 ? props.getMaxRowCountForPageSizeCheck() - : (long) rows / usedMem * remainingMem; + : rows * remainingMem / usedMem; if (rowsToFillPage < minRecordToWait) { minRecordToWait = rowsToFillPage; }