Skip to content

HIVE-26745: HPL unable to handle Decimal or null values in hplsql mode #3769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
szlta merged 2 commits into from
Nov 17, 2022
Merged

HIVE-26745: HPL unable to handle Decimal or null values in hplsql mode #3769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
34fb3d5
HIVE-26745: HPL fixes
szlta Nov 16, 2022
2f85c45
comment update
szlta Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hplsql/src/main/java/org/apache/hive/hplsql/Var.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,11 +255,11 @@ public Var setValue(QueryResult queryResult, int idx) {
cast(new Var(queryResult.column(idx, String.class)));
} else if (type == java.sql.Types.INTEGER || type == java.sql.Types.BIGINT ||
type == java.sql.Types.SMALLINT || type == java.sql.Types.TINYINT) {
cast(new Var(Long.valueOf(queryResult.column(idx, Long.class))));
cast(new Var(queryResult.column(idx, Long.class)));
} else if (type == java.sql.Types.DECIMAL || type == java.sql.Types.NUMERIC) {
cast(new Var(queryResult.column(idx, BigDecimal.class)));
} else if (type == java.sql.Types.FLOAT || type == java.sql.Types.DOUBLE) {
cast(new Var(Double.valueOf(queryResult.column(idx, Double.class))));
cast(new Var(queryResult.column(idx, Double.class)));
}
return this;
}
Expand Down
4 changes: 2 additions & 2 deletions itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
*
*/
public class TestBeeLineWithArgs {
private enum OutStream {
enum OutStream {
ERR, OUT
}

Expand Down Expand Up @@ -158,7 +158,7 @@ public static void postTests() {
* @return The stderr and stdout from running the script
* @throws Throwable
*/
private static String testCommandLineScript(List<String> argList, InputStream inputStream,
static String testCommandLineScript(List<String> argList, InputStream inputStream,
OutStream streamType)
throws Throwable {
BeeLine beeLine = new BeeLine();
Expand Down
52 changes: 36 additions & 16 deletions itests/hive-unit/src/test/java/org/apache/hive/beeline/TestHplSqlViaBeeLine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@

package org.apache.hive.beeline;

import static org.apache.hive.beeline.TestBeeLineWithArgs.OutStream;
import static org.apache.hive.beeline.TestBeeLineWithArgs.testCommandLineScript;
import static org.junit.Assert.fail;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -368,27 +368,47 @@ public void testBulkCollectFetchLoop() throws Throwable {
testScriptFile(SCRIPT_TEXT, args(), "e1=1 e2=2 e3=3 e4=4 e5=5 e6=6");
}

@Test
public void testDecimalCast() throws Throwable {
String SCRIPT_TEXT =
"DECLARE\n" +
"a DECIMAL(10,2);\n" +
"BEGIN\n" +
"SELECT CAST('10.5' AS DECIMAL(10,2)) as t INTO a;\n" +
"print (a);\n" +
"END;\n" +
"/";
testScriptFile(SCRIPT_TEXT, args(), "10.50", OutStream.ERR);
}

@Test
public void testNullCast() throws Throwable {
String SCRIPT_TEXT =
"BEGIN\n" +
"DECLARE a BIGINT;\n" +
"print('started');\n" +
"SELECT cast (null as BIGINT) as t INTO a\n" +
"print (a);\n" +
"print ('here');\n" +
"end;\n" +
"/";
// Inverted match, output should not have NPE
testScriptFile(SCRIPT_TEXT, args(), "^(.(?!(NullPointerException)))*$", OutStream.ERR);
}

private static List<String> args() {
return Arrays.asList("-d", BeeLine.BEELINE_DEFAULT_JDBC_DRIVER,
"-u", miniHS2.getBaseJdbcURL() + ";mode=hplsql", "-n", userName);
}

private static String testCommandLineScript(List<String> argList, InputStream inputStream)
throws Throwable {
BeeLine beeLine = new BeeLine();
ByteArrayOutputStream os = new ByteArrayOutputStream();
PrintStream beelineOutputStream = new PrintStream(os);
beeLine.setOutputStream(beelineOutputStream);
String[] args = argList.toArray(new String[argList.size()]);
beeLine.begin(args, inputStream);
beeLine.close();
beelineOutputStream.close();
String output = os.toString("UTF8");
return output;
}

private void testScriptFile(String scriptText, List<String> argList, String expectedPattern)
throws Throwable {
testScriptFile(scriptText, argList, expectedPattern, OutStream.OUT);
}

private void testScriptFile(String scriptText, List<String> argList, String expectedPattern,
TestBeeLineWithArgs.OutStream outStream) throws Throwable {
File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp");
scriptFile.deleteOnExit();
try (PrintStream os = new PrintStream(new FileOutputStream(scriptFile))) {
Expand All @@ -397,7 +417,7 @@ private void testScriptFile(String scriptText, List<String> argList, String expe
List<String> copy = new ArrayList<>(argList);
copy.add("-f");
copy.add(scriptFile.getAbsolutePath());
String output = testCommandLineScript(copy, null);
String output = testCommandLineScript(copy, null, outStream);
if (!Pattern.compile(".*" + expectedPattern + ".*", Pattern.DOTALL).matcher(output).matches()) {
fail("Output: '" + output + "' should match " + expectedPattern);
}
Expand Down
11 changes: 11 additions & 0 deletions service/src/java/org/apache/hive/service/cli/operation/hplsql/HplSqlQueryExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

package org.apache.hive.service.cli.operation.hplsql;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -29,6 +30,7 @@

import org.antlr.v4.runtime.ParserRuleContext;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hive.hplsql.executor.ColumnMeta;
import org.apache.hive.hplsql.executor.Metadata;
import org.apache.hive.hplsql.executor.QueryException;
Expand Down Expand Up @@ -120,6 +122,9 @@ private RowSet fetch() {

@Override
public <T> T get(int columnIndex, Class<T> type) {
if (current[columnIndex] == null) {
return null;
}
if (type.isInstance(current[columnIndex])) {
return (T) current[columnIndex];
} else {
Expand All @@ -133,6 +138,12 @@ public <T> T get(int columnIndex, Class<T> type) {
if (type == Byte.class)
return type.cast(((Number) current[columnIndex]).byteValue());
}
// RowSet can never return the HiveDecimal instances created on Hive side, nor its BigDecimal representation.
// Instead, it gets converted into String object in ColumnBasedSet.addRow()...
if (type == BigDecimal.class &&
Copy link
Contributor

@zeroflag zeroflag Nov 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this check work for the HiveDecimal case too? If yes, then LGTM.

Copy link
Contributor Author

@szlta szlta Nov 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment might be misleading. The created object on Hive side is HiveDecimal, and we expect to return a java BigDecimal instance from it. (But as stated, it turns into String first..)

serdeConstants.DECIMAL_TYPE_NAME.equalsIgnoreCase(metadata(handle).columnTypeName(columnIndex))) {
return (T) new BigDecimal((String) current[columnIndex]);
}
throw new ClassCastException(current[columnIndex].getClass() + " cannot be casted to " + type);
}
}
Expand Down