HIVE-22179 Break up DDLSemanticAnalyzer - extract Function related analyzers

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/AbstractFunctionAnalyzer.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.function;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.hooks.Entity.Type;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+/**
+ * Abstract ancestor of function related ddl analyzer classes.
+ */
+public abstract class AbstractFunctionAnalyzer extends BaseSemanticAnalyzer {
+  public AbstractFunctionAnalyzer(QueryState queryState) throws SemanticException {
+    super(queryState);
+  }
+
+  /**
+   * Add write entities to the semantic analyzer to restrict function creation to privileged users.
+   */
+  protected void addEntities(String functionName, String className, boolean isTemporary,
+      List<ResourceUri> resources) throws SemanticException {
+    // If the function is being added under a database 'namespace', then add an entity representing
+    // the database (only applicable to permanent/metastore functions).
+    // We also add a second entity representing the function name.
+    // The authorization api implementation can decide which entities it wants to use to
+    // authorize the create/drop function call.
+
+    // Add the relevant database 'namespace' as a WriteEntity
+    Database database = null;
+
+    // temporary functions don't have any database 'namespace' associated with it
+    if (!isTemporary) {
+      try {
+        String[] qualifiedNameParts = FunctionUtils.getQualifiedFunctionNameParts(functionName);
+        String databaseName = qualifiedNameParts[0];
+        functionName = qualifiedNameParts[1];
+        database = getDatabase(databaseName);
+      } catch (HiveException e) {
+        LOG.error("Failed to get database ", e);
+        throw new SemanticException(e);
+      }
+    }
+    if (database != null) {
+      outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_NO_LOCK));
+    }
+
+    // Add the function name as a WriteEntity
+    outputs.add(new WriteEntity(database, functionName, className, Type.FUNCTION, WriteEntity.WriteType.DDL_NO_LOCK));
+
+    if (resources != null) {
+      for (ResourceUri resource : resources) {
+        String uriPath = resource.getUri();
+        outputs.add(toWriteEntity(uriPath));
+      }
+    }
+  }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/create/CreateFunctionAnalyzer.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.function.create;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.ResourceType;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
+import org.apache.hadoop.hive.ql.ddl.function.AbstractFunctionAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.DDLWork;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Analyzer for function creation commands.
+ */
+@DDLType(type=HiveParser.TOK_CREATEFUNCTION)
+public class CreateFunctionAnalyzer extends AbstractFunctionAnalyzer {
+  private static final Logger SESSION_STATE_LOG = LoggerFactory.getLogger("SessionState");
+
+  public CreateFunctionAnalyzer(QueryState queryState) throws SemanticException {
+    super(queryState);
+  }
+
+  @Override
+  public void analyzeInternal(ASTNode root) throws SemanticException {
+    String functionName = root.getChild(0).getText().toLowerCase();
+    boolean isTemporary = (root.getFirstChildWithType(HiveParser.TOK_TEMPORARY) != null);
+    if (isTemporary && FunctionUtils.isQualifiedFunctionName(functionName)) {
+      throw new SemanticException("Temporary function cannot be created with a qualified name.");
+    }
+
+    String className = unescapeSQLString(root.getChild(1).getText());
+
+    List<ResourceUri> resources = getResourceList(root);
+    if (!isTemporary && resources == null) {
+      SESSION_STATE_LOG.warn("permanent functions created without USING  clause will not be replicated.");
+    }
+
+    CreateFunctionDesc desc = new CreateFunctionDesc(functionName, className, isTemporary, resources, null);
+    rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
+
+    addEntities(functionName, className, isTemporary, resources);
+  }
+
+  private static final Map<Integer, ResourceType> TOKEN_TYPE_TO_RESOURCE_TYPE = ImmutableMap.of(
+      HiveParser.TOK_JAR, ResourceType.JAR,
+      HiveParser.TOK_FILE, ResourceType.FILE,
+      HiveParser.TOK_ARCHIVE, ResourceType.ARCHIVE);
+
+  private List<ResourceUri> getResourceList(ASTNode ast) throws SemanticException {
+    List<ResourceUri> resources = null;
+
+    ASTNode resourcesNode = (ASTNode) ast.getFirstChildWithType(HiveParser.TOK_RESOURCE_LIST);
+    if (resourcesNode != null) {
+      resources = new ArrayList<ResourceUri>();
+      for (int idx = 0; idx < resourcesNode.getChildCount(); ++idx) {
+        // ^(TOK_RESOURCE_URI $resType $resPath)
+        ASTNode node = (ASTNode) resourcesNode.getChild(idx);
+        if (node.getToken().getType() != HiveParser.TOK_RESOURCE_URI) {
+          throw new SemanticException("Expected token type TOK_RESOURCE_URI but found " + node.getToken().toString());
+        }
+        if (node.getChildCount() != 2) {
+          throw new SemanticException("Expected 2 child nodes of TOK_RESOURCE_URI but found " + node.getChildCount());
+        }
+
+        ASTNode resourceTypeNode = (ASTNode) node.getChild(0);
+        ASTNode resourceUriNode = (ASTNode) node.getChild(1);
+        ResourceType resourceType = TOKEN_TYPE_TO_RESOURCE_TYPE.get(resourceTypeNode.getType());
+        if (resourceType == null) {
+          throw new SemanticException("Unexpected token " + resourceTypeNode);
+        }
+
+        resources.add(new ResourceUri(resourceType, PlanUtils.stripQuotes(resourceUriNode.getText())));
+      }
+    }
+
+    return resources;
+  }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/CreateFunctionDesc.java → ql/src/java/org/apache/hadoop/hive/ql/ddl/function/create/CreateFunctionDesc.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.ddl.function;
+package org.apache.hadoop.hive.ql.ddl.function.create;
 
 import java.io.Serializable;
 import java.util.List;

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/CreateFunctionOperation.java → ql/src/java/org/apache/hadoop/hive/ql/ddl/function/create/CreateFunctionOperation.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.ddl.function;
+package org.apache.hadoop.hive.ql.ddl.function.create;
 
 import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
 import org.apache.hadoop.hive.ql.exec.FunctionInfo;

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/create/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Function creation DDL operation. */
+package org.apache.hadoop.hive.ql.ddl.function.create;

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/desc/DescFunctionAnalyzer.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.function.desc;
+
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
+import org.apache.hadoop.hive.ql.ddl.function.AbstractFunctionAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.DDLWork;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+/**
+ * Analyzer for function describing commands.
+ */
+@DDLType(type=HiveParser.TOK_DESCFUNCTION)
+public class DescFunctionAnalyzer extends AbstractFunctionAnalyzer {
+  public DescFunctionAnalyzer(QueryState queryState) throws SemanticException {
+    super(queryState);
+  }
+
+  @Override
+  public void analyzeInternal(ASTNode root) throws SemanticException {
+    ctx.setResFile(ctx.getLocalTmpPath());
+
+    if (root.getChildCount() < 1 || root.getChildCount() > 2) {
+      throw new SemanticException("Unexpected Tokens at DESCRIBE FUNCTION");
+    }
+
+    String functionName = stripQuotes(root.getChild(0).getText());
+    boolean isExtended = root.getChildCount() == 2;
+
+    DescFunctionDesc desc = new DescFunctionDesc(ctx.getResFile(), functionName, isExtended);
+    Task<DDLWork> task = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc));
+    rootTasks.add(task);
+
+    task.setFetchSource(true);
+    setFetchTask(createFetchTask(DescFunctionDesc.SCHEMA));
+  }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/DescFunctionDesc.java → ql/src/java/org/apache/hadoop/hive/ql/ddl/function/desc/DescFunctionDesc.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.ddl.function;
+package org.apache.hadoop.hive.ql.ddl.function.desc;
 
 import java.io.Serializable;
 
@@ -34,18 +34,18 @@ public class DescFunctionDesc implements DDLDesc, Serializable {
 
   public static final String SCHEMA = "tab_name#string";
 
-  private final String resFile;
+  private final Path resFile;
   private final String name;
   private final boolean isExtended;
 
   public DescFunctionDesc(Path resFile, String name, boolean isExtended) {
-    this.resFile = resFile.toString();
+    this.resFile = resFile;
     this.name = name;
     this.isExtended = isExtended;
   }
 
   @Explain(displayName = "result file", explainLevels = { Level.EXTENDED })
-  public String getResFile() {
+  public Path getResFile() {
     return resFile;
   }
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/DescFunctionOperation.java → ql/src/java/org/apache/hadoop/hive/ql/ddl/function/desc/DescFunctionOperation.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.ddl.function;
+package org.apache.hadoop.hive.ql.ddl.function.desc;
 
 import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
 import org.apache.hadoop.hive.ql.ddl.DDLUtils;
@@ -32,7 +32,6 @@
 import java.io.IOException;
 import java.util.Set;
 
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.ddl.DDLOperation;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -48,7 +47,7 @@ public DescFunctionOperation(DDLOperationContext context, DescFunctionDesc desc)
 
   @Override
   public int execute() throws HiveException {
-    try (DataOutputStream outStream = DDLUtils.getOutputStream(new Path(desc.getResFile()), context)) {
+    try (DataOutputStream outStream = DDLUtils.getOutputStream(desc.getResFile(), context)) {
       String funcName = desc.getName();
       FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo(funcName);
       Class<?> funcClass = functionInfo == null ? null : functionInfo.getFunctionClass();

ql/src/java/org/apache/hadoop/hive/ql/ddl/function/desc/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Function describing DDL operation. */
+package org.apache.hadoop.hive.ql.ddl.function.desc;