From 801cf2163cfaf5ff9030b52c141138e18cf149bb Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Thu, 24 Jul 2025 09:34:44 -0700 Subject: [PATCH 1/3] added test cases for problematic expressions --- .../jdbc/internal/SqlParserTest.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 6c13ad365..226c02ce9 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -281,6 +281,7 @@ public Object[][] testMiscStmtDp() { {"insert into `events` (s) values ('a')", 0}, {"SELECT COUNT(*) > 0 FROM system.databases WHERE name = ?", 1}, {"SELECT count(*) > 0 FROM system.databases WHERE c1 = ?", 1}, + {"SELECT COUNT() FROM system.databases WHERE name = ?", 1}, {"alter table user delete where reg_time = ?", 1}, {"SELECT * FROM a,b WHERE id > ?", 1}, {"DROP USER IF EXISTS default_impersonation_user", 0}, @@ -304,8 +305,17 @@ public Object[][] testMiscStmtDp() { {"GRANT ON CLUSTER '{cluster}' `metabase_test_role`, `metabase-test-role` TO `metabase_test_user`", 0}, {"SELECT * FROM `test_data`.`categories` WHERE id = 1::String or id = ?", 1}, {"SELECT * FROM `test_data`.`categories` WHERE id = cast(1 as String) or id = ?", 1}, + {"select * from test_data.categories WHERE test_data.categories.name = ? limit 4", 1}, {INSERT_INLINE_DATA, 0}, {"select sum(value) from `uuid_filter_db`.`uuid_filter_table` WHERE `uuid_filter_db`.`uuid_filter_table`.`uuid` IN (CAST('36f7f85c-d7f4-49e2-af05-f45d5f6636ad' AS UUID))", 0}, + {"SELECT DISTINCT ON (column) FROM table WHER column > ?", 1}, + {"SELECT * FROM test_table \nUNION\n DISTINCT SELECT * FROM test_table", 0}, + {"SELECT * FROM test_table1 \nUNION\n SELECT * FROM test_table2 WHERE test_table2.column1 = ?", 1}, + {PARAMETRIZED_VIEW, 0}, + {COMPLEX_CTE, 0}, + {"select toYear(dt) year from test WHERE val=?", 1}, + {"select toYear(dt) AS year from test WHERE val=?", 1}, + {"select toYear(dt) AS yearx from test WHERE val=?", 1}, }; } @@ -313,4 +323,72 @@ public Object[][] testMiscStmtDp() { "INSERT INTO `interval_15_XUTLZWBLKMNZZPRZSKRF`.`checkins` (`timestamp`, `id`) " + "VALUES ((`now64`(9) + INTERVAL -225 second), 1)"; + private static final String PARAMETRIZED_VIEW = "CREATE VIEW default.test\n" + + "AS \n" + + "WITH \n" + + " toDateTime({from:String}, 'Asia/Seoul') AS FROM, \n" + + " date_add(FROM, INTERVAL 1 MINUTE) AS TO, \n" + + " {target_id:String} AS TARGET_ID \n" + + "SELECT FROM, TO, TARGET_ID"; + + private static final String COMPLEX_CTE = "WITH ? AS starting_time, ? AS ending_time, ? AS session_timeout, ? AS starting_event, ? AS ending_event, SessionData AS (\n" + + " WITH\n" + + " date,\n" + + " arraySort(\n" + + " groupArray(\n" + + " (\n" + + " tracking.event.time,\n" + + " tracking.event.event\n" + + " )\n" + + " )\n" + + " ) AS _sorted_events,\n" + + " arrayEnumerate(_sorted_events) AS _event_serial,\n" + + " arrayDifference(_sorted_events.1) AS _event_time_diff,\n" + + " \n" + + " arrayFilter(\n" + + " (x, y, z) -> y > session_timeout OR z.2 = starting_event,\n" + + " _event_serial,\n" + + " _event_time_diff,\n" + + " _sorted_events\n" + + " ) AS _gap_index_1,\n" + + "\n" + + " arrayFilter(\n" + + " (x, y) -> y.2 = ending_event,\n" + + " _event_serial,\n" + + " _sorted_events\n" + + " ) AS _gap_index_2_,\n" + + " arrayMap(\n" + + " x -> x + 1,\n" + + " _gap_index_2_\n" + + " ) AS _gap_index_2,\n" + + "\n" + + " arrayMap(x -> if (has(_gap_index_1,x) OR has(_gap_index_2,x), 1, 0), _event_serial) AS _session_splitter,\n" + + " arraySplit((x, y) -> y, _sorted_events, _session_splitter) AS _session_chain\n" + + " SELECT\n" + + " date,\n" + + " user_id AS user_id,\n" + + " arrayJoin(_session_chain) AS event_chain,\n" + + " \n" + + " arrayCompact(x -> x.2, event_chain) AS event_chain_dedup\n" + + " FROM tracking.event\n" + + " WHERE\n" + + " project=? AND time>=starting_time AND time Date: Thu, 24 Jul 2025 16:10:33 -0700 Subject: [PATCH 2/3] Fixed parser issues --- .../jdbc/internal/ClickHouseParser.g4 | 19 +++++-- .../internal/ParsedPreparedStatement.java | 5 ++ .../jdbc/PreparedStatementTest.java | 37 ++++++++++++++ .../jdbc/internal/SqlParserTest.java | 50 +++++++++++++------ 4 files changed, 91 insertions(+), 20 deletions(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 index 5c7796542..8e5f6fd4a 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 @@ -41,15 +41,20 @@ query // CTE statement ctes - : LPAREN? WITH namedQuery (',' namedQuery)* RPAREN? + : LPAREN? WITH (cteUnboundCol COMMA)* namedQuery (COMMA namedQuery)* RPAREN? ; namedQuery - : name = identifier (columnAliases)? AS '(' query ')' + : name = identifier (columnAliases)? AS LPAREN query RPAREN ; columnAliases - : '(' identifier (',' identifier)* ')' + : LPAREN? identifier (',' identifier)* RPAREN? + ; + +cteUnboundCol + : (literal AS identifier) # CteUnboundColLiteral + | (QUERY AS identifier) # CteUnboundColParam ; // ALTER statement @@ -407,7 +412,7 @@ projectionSelectStmt // SELECT statement selectUnionStmt - : selectStmtWithParens (UNION ALL selectStmtWithParens)* + : selectStmtWithParens (UNION (ALL|DISTINCT)? selectStmtWithParens)* ; selectStmtWithParens @@ -1228,6 +1233,12 @@ keywordForAlias | VIEW | PRIMARY | GRANT + | YEAR + | DAY + | MONTH + | HOUR + | MINUTE + | SECOND ; alias diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java index 794967eea..8b2496239 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java @@ -172,6 +172,11 @@ public void enterColumnExprPrecedence3(ClickHouseParser.ColumnExprPrecedence3Con super.enterColumnExprPrecedence3(ctx); } + @Override + public void enterCteUnboundColParam(ClickHouseParser.CteUnboundColParamContext ctx) { + appendParameter(ctx.start.getStartIndex()); + } + @Override public void visitErrorNode(ErrorNode node) { setHasErrors(true); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java index 2f9689b8a..35bcb696b 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/PreparedStatementTest.java @@ -1120,4 +1120,41 @@ public void testSelectWithTableAliasAsKeyword() throws Exception { } } } + + @Test(groups = {"integration"}) + public void testCTEWithUnboundCol() throws Exception { + + try (Connection conn = getJdbcConnection()) { + String cte = "with ? as text, numz as (select text, number from system.numbers limit 10) select * from numz"; + try (PreparedStatement stmt = conn.prepareStatement(cte)) { + stmt.setString(1, "1000"); + + ResultSet rs = stmt.executeQuery(); + assertTrue(rs.next()); + assertEquals(rs.getString(1), "1000"); + assertEquals(rs.getString(2), "0"); + } + } + } + + @Test(groups = {"integration"}) + public void testWithInClause() throws Exception { + + try (Connection conn = getJdbcConnection()) { + String cte = "select number from system.numbers where number in (?) limit 10"; + Long[] filter = new Long[]{2L, 4L, 6L}; + try (PreparedStatement stmt = conn.prepareStatement(cte)) { + stmt.setArray(1, conn.createArrayOf("Int64", filter)); + ResultSet rs = stmt.executeQuery(); + + for (Long filterValue : filter) { + assertTrue(rs.next()); + assertEquals(rs.getLong(1), filterValue); + } + Assert.assertFalse(rs.next()); + } + } + } + + } diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java index 226c02ce9..c2c72bf83 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/SqlParserTest.java @@ -307,15 +307,22 @@ public Object[][] testMiscStmtDp() { {"SELECT * FROM `test_data`.`categories` WHERE id = cast(1 as String) or id = ?", 1}, {"select * from test_data.categories WHERE test_data.categories.name = ? limit 4", 1}, {INSERT_INLINE_DATA, 0}, - {"select sum(value) from `uuid_filter_db`.`uuid_filter_table` WHERE `uuid_filter_db`.`uuid_filter_table`.`uuid` IN (CAST('36f7f85c-d7f4-49e2-af05-f45d5f6636ad' AS UUID))", 0}, - {"SELECT DISTINCT ON (column) FROM table WHER column > ?", 1}, - {"SELECT * FROM test_table \nUNION\n DISTINCT SELECT * FROM test_table", 0}, - {"SELECT * FROM test_table1 \nUNION\n SELECT * FROM test_table2 WHERE test_table2.column1 = ?", 1}, - {PARAMETRIZED_VIEW, 0}, - {COMPLEX_CTE, 0}, + {"select sum(value) from `uuid_filter_db`.`uuid_filter_table` WHERE `uuid_filter_db`.`uuid_filter_table`.`uuid` IN (CAST('36f7f85c-d7f4-49e2-af05-f45d5f6636ad' AS UUID))", 0}, + {"SELECT DISTINCT ON (column) FROM table WHERE column > ?", 1}, + {"SELECT * FROM test_table \nUNION\n DISTINCT SELECT * FROM test_table", 0}, + {"SELECT * FROM test_table \nUNION\n ALL SELECT * FROM test_table", 0}, + {"SELECT * FROM test_table1 \nUNION\n SELECT * FROM test_table2 WHERE test_table2.column1 = ?", 1}, + {COMPLEX_CTE, 4}, + {SIMPLE_CTE, 0}, + {CTE_CONSTANT_AS_VARIABLE, 1}, {"select toYear(dt) year from test WHERE val=?", 1}, + {"select 1 year, 2 hour, 3 minute, 4 second", 0}, {"select toYear(dt) AS year from test WHERE val=?", 1}, {"select toYear(dt) AS yearx from test WHERE val=?", 1}, + {"SELECT v FROM t WHERE f in (?)", 1}, + {"SELECT v FROM t WHERE a > 10 AND event NOT IN (?)", 1}, + {"SELECT v FROM t WHERE f in (1, 2, 3)", 0}, + {"with ? as val1, numz as (select val1, number from system.numbers limit 10) select * from numz", 1} }; } @@ -323,15 +330,7 @@ public Object[][] testMiscStmtDp() { "INSERT INTO `interval_15_XUTLZWBLKMNZZPRZSKRF`.`checkins` (`timestamp`, `id`) " + "VALUES ((`now64`(9) + INTERVAL -225 second), 1)"; - private static final String PARAMETRIZED_VIEW = "CREATE VIEW default.test\n" + - "AS \n" + - "WITH \n" + - " toDateTime({from:String}, 'Asia/Seoul') AS FROM, \n" + - " date_add(FROM, INTERVAL 1 MINUTE) AS TO, \n" + - " {target_id:String} AS TARGET_ID \n" + - "SELECT FROM, TO, TARGET_ID"; - - private static final String COMPLEX_CTE = "WITH ? AS starting_time, ? AS ending_time, ? AS session_timeout, ? AS starting_event, ? AS ending_event, SessionData AS (\n" + + private static final String COMPLEX_CTE = "WITH ? AS starting_time, ? AS ending_time, 0 AS session_timeout, '{start}' AS starting_event, '{end}' AS ending_event, SessionData AS (\n" + " WITH\n" + " date,\n" + " arraySort(\n" + @@ -373,7 +372,7 @@ public Object[][] testMiscStmtDp() { " FROM tracking.event\n" + " WHERE\n" + " project=? AND time>=starting_time AND time Date: Thu, 24 Jul 2025 16:17:45 -0700 Subject: [PATCH 3/3] made cte column in mandatory paren. --- .../antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 index 8e5f6fd4a..9187387cc 100644 --- a/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 +++ b/jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseParser.g4 @@ -49,7 +49,7 @@ namedQuery ; columnAliases - : LPAREN? identifier (',' identifier)* RPAREN? + : LPAREN identifier (',' identifier)* RPAREN ; cteUnboundCol