From c145c7847a4147cecb6a7fe517ebdd6d1217d503 Mon Sep 17 00:00:00 2001 From: Kevin Minder Date: Tue, 22 May 2018 21:46:23 -0400 Subject: [PATCH 1/9] CALCITE-2322: Add fetch size support to connection url and JDBC statement --- .../java/org/apache/calcite/avatica/AvaticaStatement.java | 2 ++ .../apache/calcite/avatica/BuiltInConnectionProperty.java | 5 ++++- .../java/org/apache/calcite/avatica/ConnectionConfig.java | 2 ++ .../org/apache/calcite/avatica/ConnectionConfigImpl.java | 4 ++++ core/src/main/java/org/apache/calcite/avatica/MetaImpl.java | 2 +- 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java index 2d3c75689a..e85161fd42 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java @@ -36,6 +36,7 @@ */ public abstract class AvaticaStatement implements Statement { + /** The default value for {@link Statement#getFetchSize()}. */ public static final int DEFAULT_FETCH_SIZE = 100; @@ -108,6 +109,7 @@ protected AvaticaStatement(AvaticaConnection connection, this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; this.resultSetHoldability = resultSetHoldability; + this.fetchSize = connection.config().fetchSize(); // Default to connection config fetch size. this.signature = signature; this.closed = false; if (h == null) { diff --git a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java index 16e1061a0e..e8fa7d1c12 100644 --- a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java @@ -86,7 +86,10 @@ public enum BuiltInConnectionProperty implements ConnectionProperty { KEY_PASSWORD("key_password", Type.STRING, "", false), HOSTNAME_VERIFICATION("hostname_verification", Type.ENUM, HostnameVerification.STRICT, - HostnameVerification.class, false); + HostnameVerification.class, false), + + /** Fetch size limit, default is 100 rows. */ + FETCH_SIZE("fetch_size", Type.NUMBER, AvaticaStatement.DEFAULT_FETCH_SIZE, false); private final String camelName; private final Type type; diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java index bbbfa87cdf..6d2a3d7731 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java @@ -62,6 +62,8 @@ public interface ConnectionConfig { String keyPassword(); /** @see BuiltInConnectionProperty#HOSTNAME_VERIFICATION */ HostnameVerification hostnameVerification(); + /** @see BuiltInConnectionProperty#FETCH_SIZE */ + int fetchSize(); } // End ConnectionConfig.java diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java index 36cdf61700..3bea18d63b 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java @@ -128,6 +128,10 @@ public HostnameVerification hostnameVerification() { .getEnum(HostnameVerification.class); } + public int fetchSize() { + return BuiltInConnectionProperty.FETCH_SIZE.wrap(properties).getInt(); + } + /** Converts a {@link Properties} object containing (name, value) * pairs into a map whose keys are * {@link org.apache.calcite.avatica.InternalProperty} objects. diff --git a/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java b/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java index 1b3b20ec55..6d0d43566e 100644 --- a/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java @@ -1589,7 +1589,7 @@ private void moveNext() { } try { // currentOffset updated after element is read from `rows` iterator - frame = fetch(stmt.handle, currentOffset, AvaticaStatement.DEFAULT_FETCH_SIZE); + frame = fetch(stmt.handle, currentOffset, stmt.getFetchSize()); } catch (NoSuchStatementException e) { resetStatement(); // re-fetch the batch where we left off From 7b52b9a7fa9d57b0c1bfcbea72891cc0828bf048 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Mon, 28 Jun 2021 16:29:23 -0700 Subject: [PATCH 2/9] re-name per feedback in ticket --- .../org/apache/calcite/avatica/BuiltInConnectionProperty.java | 4 ++-- .../java/org/apache/calcite/avatica/ConnectionConfig.java | 2 +- .../java/org/apache/calcite/avatica/ConnectionConfigImpl.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java index e8fa7d1c12..6876f328d5 100644 --- a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java @@ -88,8 +88,8 @@ public enum BuiltInConnectionProperty implements ConnectionProperty { HOSTNAME_VERIFICATION("hostname_verification", Type.ENUM, HostnameVerification.STRICT, HostnameVerification.class, false), - /** Fetch size limit, default is 100 rows. */ - FETCH_SIZE("fetch_size", Type.NUMBER, AvaticaStatement.DEFAULT_FETCH_SIZE, false); + /** The number of rows to fetch per call, default is 100 rows. */ + FETCH_ROW_COUNT("fetch_row_count", Type.NUMBER, AvaticaStatement.DEFAULT_FETCH_SIZE, false); private final String camelName; private final Type type; diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java index 6d2a3d7731..e5a25ca288 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java @@ -62,7 +62,7 @@ public interface ConnectionConfig { String keyPassword(); /** @see BuiltInConnectionProperty#HOSTNAME_VERIFICATION */ HostnameVerification hostnameVerification(); - /** @see BuiltInConnectionProperty#FETCH_SIZE */ + /** @see BuiltInConnectionProperty#FETCH_ROW_COUNT */ int fetchSize(); } diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java index 3bea18d63b..ab68793a97 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java @@ -129,7 +129,7 @@ public HostnameVerification hostnameVerification() { } public int fetchSize() { - return BuiltInConnectionProperty.FETCH_SIZE.wrap(properties).getInt(); + return BuiltInConnectionProperty.FETCH_ROW_COUNT.wrap(properties).getInt(); } /** Converts a {@link Properties} object containing (name, value) From 4e9ac288a2080ec77f13fa54c0a9811dd59e037b Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Mon, 28 Jun 2021 16:29:32 -0700 Subject: [PATCH 3/9] Add description --- site/_docs/client_reference.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/site/_docs/client_reference.md b/site/_docs/client_reference.md index f49372ab57..18b80cda63 100644 --- a/site/_docs/client_reference.md +++ b/site/_docs/client_reference.md @@ -172,3 +172,11 @@ on-hover images for the permalink, but oh well. : _Default_: `null`. : _Required_: Only if `truststore` was provided. + +fetch_row_count + +: _Description_: The number of rows to fetch + +: _Default_: `100`. + +: _Required_: No. From cb5b5431368943f822777fdd7815b5f2ab79e393 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Mon, 28 Jun 2021 16:40:33 -0700 Subject: [PATCH 4/9] update name in a few more places --- .../java/org/apache/calcite/avatica/AvaticaStatement.java | 8 ++++---- .../java/org/apache/calcite/avatica/ConnectionConfig.java | 2 +- .../org/apache/calcite/avatica/ConnectionConfigImpl.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java index e85161fd42..8f34c141bf 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java @@ -67,7 +67,7 @@ public abstract class AvaticaStatement final int resultSetType; final int resultSetConcurrency; final int resultSetHoldability; - private int fetchSize = DEFAULT_FETCH_SIZE; + private int fetchRowCount; private int fetchDirection; protected long maxRowCount = 0; @@ -109,7 +109,7 @@ protected AvaticaStatement(AvaticaConnection connection, this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; this.resultSetHoldability = resultSetHoldability; - this.fetchSize = connection.config().fetchSize(); // Default to connection config fetch size. + this.fetchRowCount = connection.config().fetchRowCount(); // Default to connection config fetch size. this.signature = signature; this.closed = false; if (h == null) { @@ -409,12 +409,12 @@ public int getFetchDirection() throws SQLException { public void setFetchSize(int rows) throws SQLException { checkOpen(); - this.fetchSize = rows; + this.fetchRowCount = rows; } public int getFetchSize() throws SQLException { checkOpen(); - return fetchSize; + return fetchRowCount; } public int getResultSetConcurrency() throws SQLException { diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java index e5a25ca288..4e1669d393 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java @@ -63,7 +63,7 @@ public interface ConnectionConfig { /** @see BuiltInConnectionProperty#HOSTNAME_VERIFICATION */ HostnameVerification hostnameVerification(); /** @see BuiltInConnectionProperty#FETCH_ROW_COUNT */ - int fetchSize(); + int fetchRowCount(); } // End ConnectionConfig.java diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java index ab68793a97..e0af59b02b 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java @@ -128,7 +128,7 @@ public HostnameVerification hostnameVerification() { .getEnum(HostnameVerification.class); } - public int fetchSize() { + public int fetchRowCount() { return BuiltInConnectionProperty.FETCH_ROW_COUNT.wrap(properties).getInt(); } From 62aeb3e0d89f1233853dfcf8b238bb728d0c0fb6 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Tue, 29 Jun 2021 09:18:17 -0700 Subject: [PATCH 5/9] fix checkstyle --- .../main/java/org/apache/calcite/avatica/AvaticaStatement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java index 8f34c141bf..36e16dcb97 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java @@ -109,7 +109,7 @@ protected AvaticaStatement(AvaticaConnection connection, this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; this.resultSetHoldability = resultSetHoldability; - this.fetchRowCount = connection.config().fetchRowCount(); // Default to connection config fetch size. + this.fetchRowCount = connection.config().fetchRowCount(); // Default to connection config value this.signature = signature; this.closed = false; if (h == null) { From d29fdce9589c60a2e33d54065efbae32f48a88c3 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Tue, 29 Jun 2021 09:18:45 -0700 Subject: [PATCH 6/9] cache value so that we only need to try/catch once --- .../main/java/org/apache/calcite/avatica/MetaImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java b/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java index 6d0d43566e..afdf237bc9 100644 --- a/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/MetaImpl.java @@ -1543,6 +1543,7 @@ public Iterator iterator() { private class FetchIterator implements Iterator { private final AvaticaStatement stmt; private final QueryState state; + private final int fetchRowCount; private Frame frame; private Iterator rows; private long currentOffset = 0; @@ -1550,6 +1551,13 @@ private class FetchIterator implements Iterator { private FetchIterator(AvaticaStatement stmt, QueryState state, Frame firstFrame) { this.stmt = stmt; this.state = state; + int fetchRowCount; + try { + fetchRowCount = stmt.getFetchSize(); + } catch (SQLException e) { + fetchRowCount = AvaticaStatement.DEFAULT_FETCH_SIZE; + } + this.fetchRowCount = fetchRowCount; if (firstFrame == null) { frame = Frame.MORE; rows = EmptyIterator.INSTANCE; @@ -1589,7 +1597,7 @@ private void moveNext() { } try { // currentOffset updated after element is read from `rows` iterator - frame = fetch(stmt.handle, currentOffset, stmt.getFetchSize()); + frame = fetch(stmt.handle, currentOffset, fetchRowCount); } catch (NoSuchStatementException e) { resetStatement(); // re-fetch the batch where we left off From b9cc9bab56dd80c3ce3281616f299c67eb565e1f Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Tue, 29 Jun 2021 09:23:38 -0700 Subject: [PATCH 7/9] attempt to clarify usage --- site/_docs/client_reference.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site/_docs/client_reference.md b/site/_docs/client_reference.md index 18b80cda63..8ec58087ac 100644 --- a/site/_docs/client_reference.md +++ b/site/_docs/client_reference.md @@ -175,7 +175,9 @@ on-hover images for the permalink, but oh well. fetch_row_count -: _Description_: The number of rows to fetch +: _Description_: The number of rows to fetch. If + + Statement:setFetchSize is set, that value overrides fetch_row_count. : _Default_: `100`. From 384a330ee2da34b5d0fa3ca57788061cb49b3839 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Tue, 29 Jun 2021 15:51:13 -0700 Subject: [PATCH 8/9] rename to fetch_size_rows --- .../java/org/apache/calcite/avatica/AvaticaStatement.java | 8 ++++---- .../apache/calcite/avatica/BuiltInConnectionProperty.java | 2 +- .../java/org/apache/calcite/avatica/ConnectionConfig.java | 4 ++-- .../org/apache/calcite/avatica/ConnectionConfigImpl.java | 4 ++-- site/_docs/client_reference.md | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java index 36e16dcb97..cb3e80140f 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java @@ -67,7 +67,7 @@ public abstract class AvaticaStatement final int resultSetType; final int resultSetConcurrency; final int resultSetHoldability; - private int fetchRowCount; + private int fetchSizeRows; private int fetchDirection; protected long maxRowCount = 0; @@ -109,7 +109,7 @@ protected AvaticaStatement(AvaticaConnection connection, this.resultSetType = resultSetType; this.resultSetConcurrency = resultSetConcurrency; this.resultSetHoldability = resultSetHoldability; - this.fetchRowCount = connection.config().fetchRowCount(); // Default to connection config value + this.fetchSizeRows = connection.config().fetchSizeRows(); // Default to connection config value this.signature = signature; this.closed = false; if (h == null) { @@ -409,12 +409,12 @@ public int getFetchDirection() throws SQLException { public void setFetchSize(int rows) throws SQLException { checkOpen(); - this.fetchRowCount = rows; + this.fetchSizeRows = rows; } public int getFetchSize() throws SQLException { checkOpen(); - return fetchRowCount; + return fetchSizeRows; } public int getResultSetConcurrency() throws SQLException { diff --git a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java index 6876f328d5..608c3cf534 100644 --- a/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java +++ b/core/src/main/java/org/apache/calcite/avatica/BuiltInConnectionProperty.java @@ -89,7 +89,7 @@ public enum BuiltInConnectionProperty implements ConnectionProperty { HostnameVerification.class, false), /** The number of rows to fetch per call, default is 100 rows. */ - FETCH_ROW_COUNT("fetch_row_count", Type.NUMBER, AvaticaStatement.DEFAULT_FETCH_SIZE, false); + FETCH_SIZE_ROWS("fetch_size_rows", Type.NUMBER, AvaticaStatement.DEFAULT_FETCH_SIZE, false); private final String camelName; private final Type type; diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java index 4e1669d393..bd4dbba72c 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfig.java @@ -62,8 +62,8 @@ public interface ConnectionConfig { String keyPassword(); /** @see BuiltInConnectionProperty#HOSTNAME_VERIFICATION */ HostnameVerification hostnameVerification(); - /** @see BuiltInConnectionProperty#FETCH_ROW_COUNT */ - int fetchRowCount(); + /** @see BuiltInConnectionProperty#FETCH_SIZE_ROWS */ + int fetchSizeRows(); } // End ConnectionConfig.java diff --git a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java index e0af59b02b..584c66814a 100644 --- a/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java +++ b/core/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java @@ -128,8 +128,8 @@ public HostnameVerification hostnameVerification() { .getEnum(HostnameVerification.class); } - public int fetchRowCount() { - return BuiltInConnectionProperty.FETCH_ROW_COUNT.wrap(properties).getInt(); + public int fetchSizeRows() { + return BuiltInConnectionProperty.FETCH_SIZE_ROWS.wrap(properties).getInt(); } /** Converts a {@link Properties} object containing (name, value) diff --git a/site/_docs/client_reference.md b/site/_docs/client_reference.md index 8ec58087ac..e24f2abdbf 100644 --- a/site/_docs/client_reference.md +++ b/site/_docs/client_reference.md @@ -173,11 +173,11 @@ on-hover images for the permalink, but oh well. : _Required_: Only if `truststore` was provided. -fetch_row_count +fetch_size_rows : _Description_: The number of rows to fetch. If - Statement:setFetchSize is set, that value overrides fetch_row_count. + Statement:setFetchSize is set, that value overrides fetch_size_rows. : _Default_: `100`. From 4dcfef15ba1f3f95bcb0d0d5553267efa0efd6b9 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 1 Jul 2021 08:17:57 -0700 Subject: [PATCH 9/9] remove blank line --- .../main/java/org/apache/calcite/avatica/AvaticaStatement.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java index cb3e80140f..5292b65ced 100644 --- a/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java +++ b/core/src/main/java/org/apache/calcite/avatica/AvaticaStatement.java @@ -36,7 +36,6 @@ */ public abstract class AvaticaStatement implements Statement { - /** The default value for {@link Statement#getFetchSize()}. */ public static final int DEFAULT_FETCH_SIZE = 100;