From df6bc4d1026b5f17feb811fd50292f81df86b71b Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Thu, 11 Aug 2011 01:25:19 -0700 Subject: [PATCH 1/2] HIVE-1360. It has been a long-standing request for UDFs to be able to access parameter values. This not only enables significant performance improvement possibilities, it also allows for fundamentally richer behavior, such as allowing the output type of a UDF to depend on its inputs. The strategy in this diff is to introduce the notion of a ConstantObjectInspector, like a regular ObjectInspector except that it encapsulates a constant value and knows what this constant value is. These COIs are created through a factory method by ExprNodeConstantDesc during plan generation hence UDFs will be able to capture these constant values during the initialize phase. Furthermore, because these ConstantObjectInspectors are simply subinterfaces of ObjectInspector, UDFs which are not "constant-aware" receive ObjectInspectors which also implement the same interfaces they are used to, so no special handling needs to be done for existing UDFs. An example UDF which uses this new functionality is also included in this diff. NAMED_STRUCT is like STRUCT except that it also allows users to specify the *names* of the fields of the struct, something previously not possible because the names of the fields must be known at compile time. --- .../ql/exec/ExprNodeConstantEvaluator.java | 18 +--- .../hadoop/hive/ql/exec/FunctionRegistry.java | 2 + .../hive/ql/plan/ExprNodeConstantDesc.java | 17 ++++ .../hadoop/hive/ql/plan/ExprNodeDesc.java | 7 ++ .../hive/ql/plan/ExprNodeGenericFuncDesc.java | 4 +- .../ql/udf/generic/GenericUDFNamedStruct.java | 89 +++++++++++++++++++ .../queries/clientpositive/udf_named_struct.q | 9 ++ .../clientpositive/udf_named_struct.q.out | 63 +++++++++++++ .../ConstantObjectInspector.java | 29 ++++++ .../PrimitiveObjectInspectorFactory.java | 43 +++++++++ ...ritableConstantBooleanObjectInspector.java | 43 +++++++++ .../WritableConstantByteObjectInspector.java | 43 +++++++++ ...WritableConstantDoubleObjectInspector.java | 43 +++++++++ .../WritableConstantFloatObjectInspector.java | 43 +++++++++ .../WritableConstantIntObjectInspector.java | 43 +++++++++ .../WritableConstantLongObjectInspector.java | 43 +++++++++ .../WritableConstantShortObjectInspector.java | 43 +++++++++ ...WritableConstantStringObjectInspector.java | 43 +++++++++ .../WritableConstantVoidObjectInspector.java | 38 ++++++++ 19 files changed, 646 insertions(+), 17 deletions(-) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java create mode 100644 ql/src/test/queries/clientpositive/udf_named_struct.q create mode 100644 ql/src/test/results/clientpositive/udf_named_struct.q.out create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java create mode 100644 serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantVoidObjectInspector.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java index af62934724c4..8842f0c6b92c 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java @@ -20,10 +20,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; /** * ExprNodeConstantEvaluator. @@ -32,19 +30,11 @@ public class ExprNodeConstantEvaluator extends ExprNodeEvaluator { protected ExprNodeConstantDesc expr; - transient ObjectInspector writableObjectInspector; - transient Object writableValue; + transient ConstantObjectInspector writableObjectInspector; public ExprNodeConstantEvaluator(ExprNodeConstantDesc expr) { this.expr = expr; - PrimitiveCategory pc = ((PrimitiveTypeInfo) expr.getTypeInfo()) - .getPrimitiveCategory(); - writableObjectInspector = PrimitiveObjectInspectorFactory - .getPrimitiveWritableObjectInspector(pc); - // Convert from Java to Writable - writableValue = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(pc).getPrimitiveWritableObject( - expr.getValue()); + writableObjectInspector = expr.getWritableObjectInspector(); } @Override @@ -54,7 +44,7 @@ public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveExcep @Override public Object evaluate(Object row) throws HiveException { - return writableValue; + return writableObjectInspector.getWritableConstantValue(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index f883daebe33f..26f2198da8de 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -164,6 +164,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMap; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMapKeys; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMapValues; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNamedStruct; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan; @@ -404,6 +405,7 @@ public final class FunctionRegistry { registerGenericUDF("array", GenericUDFArray.class); registerGenericUDF("map", GenericUDFMap.class); registerGenericUDF("struct", GenericUDFStruct.class); + registerGenericUDF("named_struct", GenericUDFNamedStruct.class); registerGenericUDF("create_union", GenericUDFUnion.class); registerGenericUDF("case", GenericUDFCase.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index bf092d4f170f..aeb7b29d47d9 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -21,6 +21,10 @@ import java.io.Serializable; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -52,6 +56,19 @@ public Object getValue() { return value; } + @Override + public ConstantObjectInspector getWritableObjectInspector() { + PrimitiveCategory pc = ((PrimitiveTypeInfo)getTypeInfo()) + .getPrimitiveCategory(); + // Convert from Java to Writable + Object writableValue = PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(pc).getPrimitiveWritableObject( + getValue()); + return PrimitiveObjectInspectorFactory + .getPrimitiveWritableConstantObjectInspector(pc, writableValue); + } + + @Override public String toString() { return "Const " + typeInfo.toString() + " " + value; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java index 6b1ba1b2fc6e..855283caf934 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** * ExprNodeDesc. @@ -64,6 +66,11 @@ public String getExprString() { return null; } + public ObjectInspector getWritableObjectInspector() { + return TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(typeInfo); + } + @Explain(displayName = "type") public String getTypeString() { return typeInfo.getTypeName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java index e264709b92d6..53bb9cdb2b45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java @@ -145,9 +145,7 @@ public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF, List children) throws UDFArgumentException { ObjectInspector[] childrenOIs = new ObjectInspector[children.size()]; for (int i = 0; i < childrenOIs.length; i++) { - childrenOIs[i] = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(children.get(i) - .getTypeInfo()); + childrenOIs[i] = children.get(i).getWritableObjectInspector(); } ObjectInspector oi = genericUDF.initialize(childrenOIs); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java new file mode 100644 index 000000000000..76b55f02b94f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import java.util.ArrayList; +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantStringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +@Description(name = "named_struct", + value = "_FUNC_(name1, val1, name2, val2, ...) - Creates a struct with the given " + + "field names and values") +public class GenericUDFNamedStruct extends GenericUDF { + Object[] ret; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + + int numFields = arguments.length; + if (numFields % 2 == 1) { + throw new UDFArgumentLengthException( + "NAMED_STRUCT expects an even number of arguments."); + } + ret = new Object[numFields / 2]; + + ArrayList fname = new ArrayList(numFields / 2); + ArrayList retOIs = new ArrayList(numFields / 2); + for (int f = 0; f < numFields; f+=2) { + if (!(arguments[f] instanceof WritableConstantStringObjectInspector)) { + throw new UDFArgumentTypeException(f, "Even arguments" + + " to NAMED_STRUCT must be a constant STRING." + arguments[f].toString()); + } + WritableConstantStringObjectInspector constantOI = + (WritableConstantStringObjectInspector)arguments[f]; + fname.add(constantOI.getWritableConstantValue().toString()); + retOIs.add(arguments[f + 1]); + } + StructObjectInspector soi = + ObjectInspectorFactory.getStandardStructObjectInspector(fname, retOIs); + return soi; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + for (int i = 0; i < arguments.length / 2; i++) { + ret[i] = arguments[2 * i + 1].get(); + } + return ret; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("named_struct("); + for (int i = 0; i < children.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(children[i]); + } + sb.append(')'); + return sb.toString(); + } +} diff --git a/ql/src/test/queries/clientpositive/udf_named_struct.q b/ql/src/test/queries/clientpositive/udf_named_struct.q new file mode 100644 index 000000000000..6168e5ca6a3b --- /dev/null +++ b/ql/src/test/queries/clientpositive/udf_named_struct.q @@ -0,0 +1,9 @@ +DESCRIBE FUNCTION named_struct; +DESCRIBE FUNCTION EXTENDED named_struct; + +EXPLAIN +SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1; + +SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1; diff --git a/ql/src/test/results/clientpositive/udf_named_struct.q.out b/ql/src/test/results/clientpositive/udf_named_struct.q.out new file mode 100644 index 000000000000..d43a44844dbb --- /dev/null +++ b/ql/src/test/results/clientpositive/udf_named_struct.q.out @@ -0,0 +1,63 @@ +PREHOOK: query: DESCRIBE FUNCTION named_struct +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION named_struct +POSTHOOK: type: DESCFUNCTION +named_struct(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values +PREHOOK: query: DESCRIBE FUNCTION EXTENDED named_struct +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION EXTENDED named_struct +POSTHOOK: type: DESCFUNCTION +named_struct(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values +PREHOOK: query: EXPLAIN +SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION named_struct "foo" 1 "bar" 2)) (TOK_SELEXPR (. (TOK_FUNCTION named_struct "foo" 1 "bar" 2) foo))) (TOK_LIMIT 1))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: named_struct('foo',1,'bar',2) + type: struct + expr: named_struct('foo',1,'bar',2).foo + type: int + outputColumnNames: _col0, _col1 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 1 + + +PREHOOK: query: SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/var/folders/C4/C40caRNsEM4C4yVangruonVUe7Y/-Tmp-/jonchang/hive_2011-08-11_01-02-24_658_503462155153078291/-mr-10000 +POSTHOOK: query: SELECT named_struct("foo", 1, "bar", 2), + named_struct("foo", 1, "bar", 2).foo FROM src LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/var/folders/C4/C40caRNsEM4C4yVangruonVUe7Y/-Tmp-/jonchang/hive_2011-08-11_01-02-24_658_503462155153078291/-mr-10000 +{"foo":1,"bar":2} 1 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java new file mode 100644 index 000000000000..ce62e0c4cfb9 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ConstantObjectInspector.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +/** + * ConstantObjectInspector. This interface should be implemented by + * ObjectInspectors which represent constant values and can return them without + * an evaluation. + */ +public interface ConstantObjectInspector extends ObjectInspector { + + Object getWritableConstantValue(); + +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 8cb7983bcb25..6861ab155f50 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -20,10 +20,19 @@ import java.util.HashMap; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; +import org.apache.hadoop.hive.serde2.io.ShortWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.Text; /** * PrimitiveObjectInspectorFactory is the primary way to create new @@ -136,6 +145,40 @@ public static AbstractPrimitiveWritableObjectInspector getPrimitiveWritableObjec return result; } + /** + * Returns a PrimitiveWritableObjectInspector which implements ConstantObjectInspector + * for the PrimitiveCategory. + * + * @param primitiveCategory + * @param value + */ + public static ConstantObjectInspector getPrimitiveWritableConstantObjectInspector( + PrimitiveCategory primitiveCategory, Object value) { + switch (primitiveCategory) { + case BOOLEAN: + return new WritableConstantBooleanObjectInspector((BooleanWritable)value); + case BYTE: + return new WritableConstantByteObjectInspector((ByteWritable)value); + case SHORT: + return new WritableConstantShortObjectInspector((ShortWritable)value); + case INT: + return new WritableConstantIntObjectInspector((IntWritable)value); + case LONG: + return new WritableConstantLongObjectInspector((LongWritable)value); + case FLOAT: + return new WritableConstantFloatObjectInspector((FloatWritable)value); + case DOUBLE: + return new WritableConstantDoubleObjectInspector((DoubleWritable)value); + case STRING: + return new WritableConstantStringObjectInspector((Text)value); + case VOID: + return new WritableConstantVoidObjectInspector(); + default: + throw new RuntimeException("Internal error: Cannot find " + + "ConstantObjectInspector for " + primitiveCategory); + } + } + /** * Returns the PrimitiveJavaObjectInspector for the PrimitiveCategory. * diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java new file mode 100644 index 000000000000..03a92e8063a4 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantBooleanObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.BooleanWritable; + +/** + * A WritableConstantBooleanObjectInspector is a WritableBooleanObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantBooleanObjectInspector extends + WritableBooleanObjectInspector implements + ConstantObjectInspector { + + private BooleanWritable value; + + WritableConstantBooleanObjectInspector(BooleanWritable value) { + super(); + this.value = value; + } + + @Override + public BooleanWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java new file mode 100644 index 000000000000..5a7fe537d072 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.ByteWritable; + +/** + * A WritableConstantByteObjectInspector is a WritableByteObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantByteObjectInspector extends + WritableByteObjectInspector implements + ConstantObjectInspector { + + private ByteWritable value; + + WritableConstantByteObjectInspector(ByteWritable value) { + super(); + this.value = value; + } + + @Override + public ByteWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java new file mode 100644 index 000000000000..eedade1fba3f --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.DoubleWritable; + +/** + * A WritableConstantDoubleObjectInspector is a WritableDoubleObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantDoubleObjectInspector extends + WritableDoubleObjectInspector implements + ConstantObjectInspector { + + private DoubleWritable value; + + WritableConstantDoubleObjectInspector(DoubleWritable value) { + super(); + this.value = value; + } + + @Override + public DoubleWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java new file mode 100644 index 000000000000..d40bdc75e626 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantFloatObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.FloatWritable; + +/** + * A WritableConstantFloatObjectInspector is a WritableFloatObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantFloatObjectInspector extends + WritableFloatObjectInspector implements + ConstantObjectInspector { + + private FloatWritable value; + + WritableConstantFloatObjectInspector(FloatWritable value) { + super(); + this.value = value; + } + + @Override + public FloatWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java new file mode 100644 index 000000000000..8cb8c1c147d9 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantIntObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.IntWritable; + +/** + * A WritableConstantIntObjectInspector is a WritableIntObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantIntObjectInspector extends + WritableIntObjectInspector implements + ConstantObjectInspector { + + private IntWritable value; + + WritableConstantIntObjectInspector(IntWritable value) { + super(); + this.value = value; + } + + @Override + public IntWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java new file mode 100644 index 000000000000..42f9eed4bae6 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantLongObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.LongWritable; + +/** + * A WritableConstantLongObjectInspector is a WritableLongObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantLongObjectInspector extends + WritableLongObjectInspector implements + ConstantObjectInspector { + + private LongWritable value; + + WritableConstantLongObjectInspector(LongWritable value) { + super(); + this.value = value; + } + + @Override + public LongWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java new file mode 100644 index 000000000000..a476fa04c216 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantShortObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.hive.serde2.io.ShortWritable; + +/** + * A WritableConstantShortObjectInspector is a WritableShortObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantShortObjectInspector extends + WritableShortObjectInspector implements + ConstantObjectInspector { + + private ShortWritable value; + + WritableConstantShortObjectInspector(ShortWritable value) { + super(); + this.value = value; + } + + @Override + public ShortWritable getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java new file mode 100644 index 000000000000..802357b612e6 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantStringObjectInspector.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +import org.apache.hadoop.io.Text; + +/** + * A WritableConstantStringObjectInspector is a WritableStringObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantStringObjectInspector extends + WritableStringObjectInspector implements + ConstantObjectInspector { + + private Text value; + + WritableConstantStringObjectInspector(Text value) { + super(); + this.value = value; + } + + @Override + public Text getWritableConstantValue() { + return value; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantVoidObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantVoidObjectInspector.java new file mode 100644 index 000000000000..f3aee4e5b446 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantVoidObjectInspector.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector.primitive; + +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; + +/** + * A WritableConstantVoidObjectInspector is a WritableVoidObjectInspector + * that implements ConstantObjectInspector. + */ +public class WritableConstantVoidObjectInspector extends + WritableVoidObjectInspector implements + ConstantObjectInspector { + + WritableConstantVoidObjectInspector() { + super(); + } + + @Override + public Object getWritableConstantValue() { + return null; + } +} From 6122438708e3054baa24ce0e13d12f9405272fb0 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Thu, 11 Aug 2011 01:25:19 -0700 Subject: [PATCH 2/2] HIVE-1360. It has been a long-standing request for UDFs to be able to access parameter values. This not only enables significant performance improvement possibilities, it also allows for fundamentally richer behavior, such as allowing the output type of a UDF to depend on its inputs. The strategy in this diff is to introduce the notion of a ConstantObjectInspector, like a regular ObjectInspector except that it encapsulates a constant value and knows what this constant value is. These COIs are created through a factory method by ExprNodeConstantDesc during plan generation hence UDFs will be able to capture these constant values during the initialize phase. Furthermore, because these ConstantObjectInspectors are simply subinterfaces of ObjectInspector, UDFs which are not "constant-aware" receive ObjectInspectors which also implement the same interfaces they are used to, so no special handling needs to be done for existing UDFs. An example UDF which uses this new functionality is also included in this diff. NAMED_STRUCT is like STRUCT except that it also allows users to specify the *names* of the fields of the struct, something previously not possible because the names of the fields must be known at compile time. --- ql/src/test/results/clientpositive/show_functions.q.out | 1 + .../primitive/PrimitiveObjectInspectorFactory.java | 2 +- .../primitive/WritableConstantDoubleObjectInspector.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out index a194779a2fef..51a768d4ad7a 100644 --- a/ql/src/test/results/clientpositive/show_functions.q.out +++ b/ql/src/test/results/clientpositive/show_functions.q.out @@ -97,6 +97,7 @@ max min minute month +named_struct negative ngrams not diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java index 6861ab155f50..9aa7bddb9301 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java @@ -31,7 +31,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.FloatWritable; -import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.Text; /** diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java index eedade1fba3f..4fe163978dfd 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantDoubleObjectInspector.java @@ -19,7 +19,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; -import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** * A WritableConstantDoubleObjectInspector is a WritableDoubleObjectInspector