From be21598cb7ad3ba70a9ca6b322be7b1765425e63 Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Mon, 16 May 2016 14:58:14 +0900 Subject: [PATCH 1/3] TAJO-2158: The concat_ws function can't support a tab separator. --- .../TestStringOperatorsAndFunctions.java | 17 +++++++++++------ .../tajo/engine/function/string/Concat_ws.java | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java index a38eb71858..87c642cbc1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java @@ -316,11 +316,11 @@ public void testDigest() throws TajoException { testSimpleEval("select digest('tajo', 'md2') as col1 ", new String[]{"bf523bce8241982f6bea9af0f7fd37ff"}); testSimpleEval("select digest('tajo', 'md5') as col1 ", new String[]{"742721b3a79f71a9491681b8e8a7ce85"}); testSimpleEval("select digest('tajo', 'sha1') as col1 ", new String[]{"02b0e20540b89f0b735092bbac8093eb2e3804cf"}); - testSimpleEval("select digest('tajo', 'sha256') as col1 ", + testSimpleEval("select digest('tajo', 'sha256') as col1 ", new String[]{"6440083be076869a9f9d0271a4bf298d98c8aa3ecb49df841895fbcddbb04a70"}); - testSimpleEval("select digest('tajo', 'sha384') as col1 ", + testSimpleEval("select digest('tajo', 'sha384') as col1 ", new String[]{"59ff99b0e274eb3d8e10f221b6b949bfc1244d2a1226c5c720062fb03d82272be633e4a0f2babccffbfdff7cc1cb06fb"}); - testSimpleEval("select digest('tajo', 'sha512') as col1 ", + testSimpleEval("select digest('tajo', 'sha512') as col1 ", new String[]{"ee8ba254d331ddfb1bca9aaf0c4b8c58aea5331928cbd20168c87828afb853b0c096af71ec69a23b669217a1dddd2934edaac33b1296fe526b22abd28a15c4b3"}); testSimpleEval("select digest('tajo', 'not') as col1 ", new String[]{NullDatum.get().toString()}); } @@ -417,7 +417,7 @@ public void testSubstr() throws TajoException { testEval(schema, "table1", ",abcdef,3.14", "select substr(lower(col2), 2, 3) from table1", new String[]{"bcd"}); } - + @Test public void testLocate() throws TajoException { // normal case @@ -448,8 +448,8 @@ public void testLocate() throws TajoException { testSimpleEval("select locate('가나다라', '', 4) as col1 ", new String[]{"4"}); testSimpleEval("select locate('가나다라', '', 5) as col1 ", new String[]{"5"}); testSimpleEval("select locate('가나다라', '', 6) as col1 ", new String[]{"0"}); - - // negative pos + + // negative pos testSimpleEval("select locate('abcdef', 'a', -1) as col1 ", new String[]{"0"}); testSimpleEval("select locate('abcdef', 'a', -5) as col1 ", new String[]{"0"}); @@ -639,5 +639,10 @@ public void testConcat_ws() throws TajoException { testSimpleEval("select concat_ws(',', '22', null) ", new String[]{"22"}); testSimpleEval("select concat_ws(',', '22', '33', '33') ", new String[]{"22,33,33"}); testSimpleEval("select concat_ws(',', null, '22') ", new String[]{"22"}); + + testSimpleEval("select concat_ws('\t', '한글', '22') ", new String[]{"한글\t22"}); + testSimpleEval("select concat_ws('\t', '22', null) ", new String[]{"22"}); + testSimpleEval("select concat_ws('\t', '22', '33', '33') ", new String[]{"22\t33\t33"}); + testSimpleEval("select concat_ws('\t', null, '22') ", new String[]{"22"}); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java index bbddeb16d6..c160f24738 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.function.string; import com.google.gson.annotations.Expose; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; @@ -60,7 +61,7 @@ public Datum eval(Tuple params) { return NullDatum.get(); } - String separator = params.getText(0); + String separator = StringEscapeUtils.unescapeJava(params.getText(0)); StringBuilder result = new StringBuilder(); From 65aa34d372adcfc6c82d5b7f14503389f2ef636b Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Mon, 16 May 2016 15:00:49 +0900 Subject: [PATCH 2/3] TAJO-2158: The concat_ws function can't support a tab separator. --- .../function/TestStringOperatorsAndFunctions.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java index 87c642cbc1..7a2f12a338 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java @@ -316,11 +316,11 @@ public void testDigest() throws TajoException { testSimpleEval("select digest('tajo', 'md2') as col1 ", new String[]{"bf523bce8241982f6bea9af0f7fd37ff"}); testSimpleEval("select digest('tajo', 'md5') as col1 ", new String[]{"742721b3a79f71a9491681b8e8a7ce85"}); testSimpleEval("select digest('tajo', 'sha1') as col1 ", new String[]{"02b0e20540b89f0b735092bbac8093eb2e3804cf"}); - testSimpleEval("select digest('tajo', 'sha256') as col1 ", + testSimpleEval("select digest('tajo', 'sha256') as col1 ", new String[]{"6440083be076869a9f9d0271a4bf298d98c8aa3ecb49df841895fbcddbb04a70"}); - testSimpleEval("select digest('tajo', 'sha384') as col1 ", + testSimpleEval("select digest('tajo', 'sha384') as col1 ", new String[]{"59ff99b0e274eb3d8e10f221b6b949bfc1244d2a1226c5c720062fb03d82272be633e4a0f2babccffbfdff7cc1cb06fb"}); - testSimpleEval("select digest('tajo', 'sha512') as col1 ", + testSimpleEval("select digest('tajo', 'sha512') as col1 ", new String[]{"ee8ba254d331ddfb1bca9aaf0c4b8c58aea5331928cbd20168c87828afb853b0c096af71ec69a23b669217a1dddd2934edaac33b1296fe526b22abd28a15c4b3"}); testSimpleEval("select digest('tajo', 'not') as col1 ", new String[]{NullDatum.get().toString()}); } @@ -417,7 +417,7 @@ public void testSubstr() throws TajoException { testEval(schema, "table1", ",abcdef,3.14", "select substr(lower(col2), 2, 3) from table1", new String[]{"bcd"}); } - + @Test public void testLocate() throws TajoException { // normal case @@ -448,8 +448,8 @@ public void testLocate() throws TajoException { testSimpleEval("select locate('가나다라', '', 4) as col1 ", new String[]{"4"}); testSimpleEval("select locate('가나다라', '', 5) as col1 ", new String[]{"5"}); testSimpleEval("select locate('가나다라', '', 6) as col1 ", new String[]{"0"}); - - // negative pos + + // negative pos testSimpleEval("select locate('abcdef', 'a', -1) as col1 ", new String[]{"0"}); testSimpleEval("select locate('abcdef', 'a', -5) as col1 ", new String[]{"0"}); From 107fd449316dcc4142ec5c3fd98f884c8d7b5f2f Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Mon, 16 May 2016 15:03:23 +0900 Subject: [PATCH 3/3] TAJO-2158: The concat_ws function can't support a tab separator. --- .../tajo/engine/function/TestStringOperatorsAndFunctions.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java index 7a2f12a338..e266409d67 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/function/TestStringOperatorsAndFunctions.java @@ -640,6 +640,7 @@ public void testConcat_ws() throws TajoException { testSimpleEval("select concat_ws(',', '22', '33', '33') ", new String[]{"22,33,33"}); testSimpleEval("select concat_ws(',', null, '22') ", new String[]{"22"}); + testSimpleEval("select concat_ws('\t', '333', '22') ", new String[]{"333\t22"}); testSimpleEval("select concat_ws('\t', '한글', '22') ", new String[]{"한글\t22"}); testSimpleEval("select concat_ws('\t', '22', null) ", new String[]{"22"}); testSimpleEval("select concat_ws('\t', '22', '33', '33') ", new String[]{"22\t33\t33"});