From e59e0ad812c216ed1cdd3af410a6e37f0b3da47e Mon Sep 17 00:00:00 2001 From: lvca Date: Fri, 20 Dec 2024 11:06:25 -0500 Subject: [PATCH 1/3] fix: used Binary with dynamic allocation instead of ByteBuffer --- .../postgres/PostgresNetworkExecutor.java | 43 +++++++------------ .../com/arcadedb/postgres/PostgresType.java | 5 ++- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java b/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java index c9368c4dd4..221a33380d 100755 --- a/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java +++ b/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java @@ -20,6 +20,7 @@ import com.arcadedb.Constants; import com.arcadedb.GlobalConfiguration; +import com.arcadedb.database.Binary; import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseContext; import com.arcadedb.database.DatabaseFactory; @@ -44,26 +45,14 @@ import com.arcadedb.utility.FileUtils; import com.arcadedb.utility.Pair; -import java.io.EOFException; -import java.io.IOException; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Level; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.io.*; +import java.net.*; +import java.nio.*; +import java.nio.charset.*; +import java.util.*; +import java.util.concurrent.*; +import java.util.logging.*; +import java.util.regex.*; /** * Postgres Reference for Protocol Messages: https://www.postgresql.org/docs/9.6/protocol-message-formats.html @@ -497,8 +486,8 @@ private void writeDataRows(final List resultSet, final Map %d row data (%s) (thread=%s)", resultSet.size(), - FileUtils.getSizeAsString(bufferData.limit()), Thread.currentThread().getId()); + FileUtils.getSizeAsString(bufferData.getByteBuffer().limit()), Thread.currentThread().getId()); } private void bindCommand() { diff --git a/postgresw/src/main/java/com/arcadedb/postgres/PostgresType.java b/postgresw/src/main/java/com/arcadedb/postgres/PostgresType.java index 0cb73d4732..de48874c1d 100644 --- a/postgresw/src/main/java/com/arcadedb/postgres/PostgresType.java +++ b/postgresw/src/main/java/com/arcadedb/postgres/PostgresType.java @@ -18,6 +18,7 @@ */ package com.arcadedb.postgres; +import com.arcadedb.database.Binary; import com.arcadedb.database.DatabaseFactory; import java.nio.*; @@ -110,7 +111,7 @@ public enum PostgresType { // } // } - public void serializeAsText(final long code, final ByteBuffer typeBuffer, Object value) { + public void serializeAsText(final long code, final Binary typeBuffer, Object value) { if (value == null) { if (code == BOOLEAN.code) value = "0"; @@ -122,7 +123,7 @@ public void serializeAsText(final long code, final ByteBuffer typeBuffer, Object final byte[] str = value.toString().getBytes(DatabaseFactory.getDefaultCharset()); typeBuffer.putInt(str.length); - typeBuffer.put(str); + typeBuffer.putByteArray(str); } public static Object deserialize(final long code, final int formatCode, final byte[] valueAsBytes) { From 67553845fe74c83c18aa2b41e685a0bb125fe0e5 Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Fri, 20 Dec 2024 18:41:33 +0100 Subject: [PATCH 2/3] Remove use of ByteBuffer in favour of Binary --- .../postgres/PostgresNetworkExecutor.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java b/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java index 221a33380d..8461084c53 100755 --- a/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java +++ b/postgresw/src/main/java/com/arcadedb/postgres/PostgresNetworkExecutor.java @@ -45,14 +45,24 @@ import com.arcadedb.utility.FileUtils; import com.arcadedb.utility.Pair; -import java.io.*; -import java.net.*; -import java.nio.*; -import java.nio.charset.*; -import java.util.*; -import java.util.concurrent.*; -import java.util.logging.*; -import java.util.regex.*; +import java.io.EOFException; +import java.io.IOException; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Postgres Reference for Protocol Messages: https://www.postgresql.org/docs/9.6/protocol-message-formats.html @@ -453,14 +463,15 @@ private void writeRowDescription(final Map columns) { if (columns == null) return; - final ByteBuffer bufferDescription = ByteBuffer.allocate(64 * 1024).order(ByteOrder.BIG_ENDIAN); +// final ByteBuffer bufferDescription = ByteBuffer.allocate(64 * 1024).order(ByteOrder.BIG_ENDIAN); + final Binary bufferDescription = new Binary(); for (final Map.Entry col : columns.entrySet()) { final String columnName = col.getKey(); final PostgresType columnType = col.getValue(); - bufferDescription.put(columnName.getBytes(DatabaseFactory.getDefaultCharset()));//The field name. - bufferDescription.put((byte) 0); + bufferDescription.putByteArray(columnName.getBytes(DatabaseFactory.getDefaultCharset()));//The field name. + bufferDescription.putByte((byte) 0); bufferDescription.putInt( 0); //If the field can be identified as a column of a specific table, the object ID of the table; otherwise zero. @@ -478,8 +489,8 @@ private void writeRowDescription(final Map columns) { bufferDescription.flip(); writeMessage("row description", () -> { channel.writeUnsignedShort((short) columns.size()); - channel.writeBuffer(bufferDescription); - }, 'T', 4 + 2 + bufferDescription.limit()); + channel.writeBuffer(bufferDescription.getByteBuffer()); + }, 'T', 4 + 2 + bufferDescription.capacity()); } private void writeDataRows(final List resultSet, final Map columns) throws IOException { From 23ca89892e7e88e1964a783184b99d295358e8d1 Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Fri, 20 Dec 2024 18:57:00 +0100 Subject: [PATCH 3/3] Add test with big result set using JDBC --- .../com/arcadedb/e2e/JdbcQueriesTest.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/e2e/src/test/java/com/arcadedb/e2e/JdbcQueriesTest.java b/e2e/src/test/java/com/arcadedb/e2e/JdbcQueriesTest.java index 51fc44c83d..fc38ae8bff 100644 --- a/e2e/src/test/java/com/arcadedb/e2e/JdbcQueriesTest.java +++ b/e2e/src/test/java/com/arcadedb/e2e/JdbcQueriesTest.java @@ -23,8 +23,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.sql.*; -import java.util.*; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; import static org.assertj.core.api.Assertions.assertThat; @@ -67,6 +71,19 @@ void simpleSQLQuery() throws Exception { } } + @Test + void bigResultSetSQLQuery() throws Exception { + + try (final Statement st = conn.createStatement()) { + + try (final ResultSet rs = st.executeQuery("SELECT * FROM Beer limit -1")) { + while (rs.next()) { + assertThat(rs.getString("name")).isNotBlank(); + } + } + } + } + @Test void simpleGremlinQuery() throws Exception { try (final Statement st = conn.createStatement()) {