diff --git a/codestyle/spotbugs-exclude.xml b/codestyle/spotbugs-exclude.xml index ae93e382cea8..b35b138d3dfe 100644 --- a/codestyle/spotbugs-exclude.xml +++ b/codestyle/spotbugs-exclude.xml @@ -28,6 +28,17 @@ Reference: https://github.com/apache/druid/pull/7894/files --> + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 385f45d6d374..bf98f4ea74bb 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,8 @@ 2.2.4 1.17.0 1.9.2 + 1.21.0 3.0.0 2.0.0 @@ -1314,6 +1316,9 @@ **/SomeAvroDatum.class + **/DruidSqlParserImpl.class + **/DruidSqlParserImplTokenManager.class + **/SimpleCharStream.class **.SuppressForbidden @@ -1501,7 +1506,12 @@ - + + com.googlecode.fmpp-maven-plugin + fmpp-maven-plugin + 1.0 + + org.apache.maven.plugins maven-surefire-plugin 2.22.2 @@ -1601,6 +1611,16 @@ exec-maven-plugin 1.6.0 + + org.codehaus.mojo + javacc-maven-plugin + 2.4 + + + org.codehaus.mojo + build-helper-maven-plugin + 3.1.0 + org.apache.maven.plugins maven-javadoc-plugin @@ -1888,6 +1908,7 @@ **/dependency-reduced-pom.xml .editorconfig **/hadoop.indexer.libs.version + **/codegen/** diff --git a/sql/pom.xml b/sql/pom.xml index 5b402c762102..7855dff16bc9 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -180,6 +180,11 @@ validation-api provided + + org.slf4j + slf4j-api + provided + @@ -255,7 +260,126 @@ + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-parser-template + initialize + + unpack + + + + + org.apache.calcite + calcite-core + ${calcite.version} + jar + true + ${project.build.directory}/ + **/Parser.jj + + + + + + + + + + maven-resources-plugin + + + copy-fmpp-resources + initialize + + copy-resources + + + ${project.build.directory}/codegen + + + src/main/codegen + false + + + + + + + + + + com.googlecode.fmpp-maven-plugin + fmpp-maven-plugin + + + generate-fmpp-sources + generate-sources + + generate + + + ${project.build.directory}/codegen/config.fmpp + ${project.build.directory}/generated-sources/annotations + ${project.build.directory}/codegen/templates + + + + + + + + org.codehaus.mojo + javacc-maven-plugin + + + generate-sources + javacc + + javacc + + + ${project.build.directory}/generated-sources/annotations + + **/Parser.jj + + 2 + false + ${project.build.directory}/generated-sources/annotations + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + generate-sources + + add-source + + + + src/generated-sources/annotations + + + + + + diff --git a/sql/src/main/codegen/config.fmpp b/sql/src/main/codegen/config.fmpp new file mode 100644 index 000000000000..d751b62b9d67 --- /dev/null +++ b/sql/src/main/codegen/config.fmpp @@ -0,0 +1,447 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is an FMPP (http://fmpp.sourceforge.net/) configuration file to +# allow clients to extend Calcite's SQL parser to support application specific +# SQL statements, literals or data types. +# +# Calcite's parser grammar file (Parser.jj) is written in javacc +# (http://javacc.java.net/) with Freemarker (http://freemarker.org/) variables +# to allow clients to: +# 1. have custom parser implementation class and package name. +# 2. insert new parser method implementations written in javacc to parse +# custom: +# a) SQL statements. +# b) literals. +# c) data types. +# 3. add new keywords to support custom SQL constructs added as part of (2). +# 4. add import statements needed by inserted custom parser implementations. +# +# Parser template file (Parser.jj) along with this file are packaged as +# part of the calcite-core-.jar under "codegen" directory. + +# This file is directly copied from calite-core-1.21.0.jar/codegen/config.fmpp, and then modified slightly. +# While not a necessary requirement, it would be ideal if it is kept in line with calcite-core's version. In the newer +# Calcite versions, there is a default_config.fmpp which will free us from maintaining this file. +# +# Following clauses are modified in the file: +# 1. data.parser.package & data.parser.class +# 2. data.parser.imports +# 3. data.parser.keywords (Added "CLUSTERED", "PARTITIONED") +# 4. data.parser.statementParserMethods +# 5. data.parser.implementationFiles +data: { + parser: { + # Generated parser implementation package and class name. + package: "org.apache.druid.sql.calcite.parser", + class: "DruidSqlParserImpl", + + # List of additional classes and packages to import. + # Example. "org.apache.calcite.sql.*", "java.util.List". + imports: [ + "org.apache.calcite.sql.SqlNode" + "org.apache.calcite.sql.SqlInsert" + "org.apache.druid.java.util.common.granularity.Granularity" + "org.apache.druid.java.util.common.granularity.Granularities" + "org.apache.druid.sql.calcite.parser.DruidSqlInsert" + "org.apache.druid.sql.calcite.parser.DruidSqlParserUtils" + ] + + # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is not a reserved + # keyword add it to 'nonReservedKeywords' section. + keywords: [ + "CLUSTERED" + "PARTITIONED" + ] + + # List of keywords from "keywords" section that are not reserved. + nonReservedKeywords: [ + "A" + "ABSENT" + "ABSOLUTE" + "ACTION" + "ADA" + "ADD" + "ADMIN" + "AFTER" + "ALWAYS" + "APPLY" + "ASC" + "ASSERTION" + "ASSIGNMENT" + "ATTRIBUTE" + "ATTRIBUTES" + "BEFORE" + "BERNOULLI" + "BREADTH" + "C" + "CASCADE" + "CATALOG" + "CATALOG_NAME" + "CENTURY" + "CHAIN" + "CHARACTER_SET_CATALOG" + "CHARACTER_SET_NAME" + "CHARACTER_SET_SCHEMA" + "CHARACTERISTICS" + "CHARACTERS" + "CLASS_ORIGIN" + "COBOL" + "COLLATION" + "COLLATION_CATALOG" + "COLLATION_NAME" + "COLLATION_SCHEMA" + "COLUMN_NAME" + "COMMAND_FUNCTION" + "COMMAND_FUNCTION_CODE" + "COMMITTED" + "CONDITION_NUMBER" + "CONDITIONAL" + "CONNECTION" + "CONNECTION_NAME" + "CONSTRAINT_CATALOG" + "CONSTRAINT_NAME" + "CONSTRAINT_SCHEMA" + "CONSTRAINTS" + "CONSTRUCTOR" + "CONTINUE" + "CURSOR_NAME" + "DATA" + "DATABASE" + "DATETIME_INTERVAL_CODE" + "DATETIME_INTERVAL_PRECISION" + "DECADE" + "DEFAULTS" + "DEFERRABLE" + "DEFERRED" + "DEFINED" + "DEFINER" + "DEGREE" + "DEPTH" + "DERIVED" + "DESC" + "DESCRIPTION" + "DESCRIPTOR" + "DIAGNOSTICS" + "DISPATCH" + "DOMAIN" + "DOW" + "DOY" + "DYNAMIC_FUNCTION" + "DYNAMIC_FUNCTION_CODE" + "ENCODING" + "EPOCH" + "ERROR" + "EXCEPTION" + "EXCLUDE" + "EXCLUDING" + "FINAL" + "FIRST" + "FOLLOWING" + "FORMAT" + "FORTRAN" + "FOUND" + "FRAC_SECOND" + "G" + "GENERAL" + "GENERATED" + "GEOMETRY" + "GO" + "GOTO" + "GRANTED" + "HIERARCHY" + "IGNORE" + "IMMEDIATE" + "IMMEDIATELY" + "IMPLEMENTATION" + "INCLUDING" + "INCREMENT" + "INITIALLY" + "INPUT" + "INSTANCE" + "INSTANTIABLE" + "INVOKER" + "ISODOW" + "ISOYEAR" + "ISOLATION" + "JAVA" + "JSON" + "K" + "KEY" + "KEY_MEMBER" + "KEY_TYPE" + "LABEL" + "LAST" + "LENGTH" + "LEVEL" + "LIBRARY" + "LOCATOR" + "M" + "MAP" + "MATCHED" + "MAXVALUE" + "MICROSECOND" + "MESSAGE_LENGTH" + "MESSAGE_OCTET_LENGTH" + "MESSAGE_TEXT" + "MILLISECOND" + "MILLENNIUM" + "MINVALUE" + "MORE_" + "MUMPS" + "NAME" + "NAMES" + "NANOSECOND" + "NESTING" + "NORMALIZED" + "NULLABLE" + "NULLS" + "NUMBER" + "OBJECT" + "OCTETS" + "OPTION" + "OPTIONS" + "ORDERING" + "ORDINALITY" + "OTHERS" + "OUTPUT" + "OVERRIDING" + "PAD" + "PARAMETER_MODE" + "PARAMETER_NAME" + "PARAMETER_ORDINAL_POSITION" + "PARAMETER_SPECIFIC_CATALOG" + "PARAMETER_SPECIFIC_NAME" + "PARAMETER_SPECIFIC_SCHEMA" + "PARTIAL" + "PASCAL" + "PASSING" + "PASSTHROUGH" + "PAST" + "PATH" + "PLACING" + "PLAN" + "PLI" + "PRECEDING" + "PRESERVE" + "PRIOR" + "PRIVILEGES" + "PUBLIC" + "QUARTER" + "READ" + "RELATIVE" + "REPEATABLE" + "REPLACE" + "RESPECT" + "RESTART" + "RESTRICT" + "RETURNED_CARDINALITY" + "RETURNED_LENGTH" + "RETURNED_OCTET_LENGTH" + "RETURNED_SQLSTATE" + "RETURNING" + "ROLE" + "ROUTINE" + "ROUTINE_CATALOG" + "ROUTINE_NAME" + "ROUTINE_SCHEMA" + "ROW_COUNT" + "SCALAR" + "SCALE" + "SCHEMA" + "SCHEMA_NAME" + "SCOPE_CATALOGS" + "SCOPE_NAME" + "SCOPE_SCHEMA" + "SECTION" + "SECURITY" + "SELF" + "SEQUENCE" + "SERIALIZABLE" + "SERVER" + "SERVER_NAME" + "SESSION" + "SETS" + "SIMPLE" + "SIZE" + "SOURCE" + "SPACE" + "SPECIFIC_NAME" + "SQL_BIGINT" + "SQL_BINARY" + "SQL_BIT" + "SQL_BLOB" + "SQL_BOOLEAN" + "SQL_CHAR" + "SQL_CLOB" + "SQL_DATE" + "SQL_DECIMAL" + "SQL_DOUBLE" + "SQL_FLOAT" + "SQL_INTEGER" + "SQL_INTERVAL_DAY" + "SQL_INTERVAL_DAY_TO_HOUR" + "SQL_INTERVAL_DAY_TO_MINUTE" + "SQL_INTERVAL_DAY_TO_SECOND" + "SQL_INTERVAL_HOUR" + "SQL_INTERVAL_HOUR_TO_MINUTE" + "SQL_INTERVAL_HOUR_TO_SECOND" + "SQL_INTERVAL_MINUTE" + "SQL_INTERVAL_MINUTE_TO_SECOND" + "SQL_INTERVAL_MONTH" + "SQL_INTERVAL_SECOND" + "SQL_INTERVAL_YEAR" + "SQL_INTERVAL_YEAR_TO_MONTH" + "SQL_LONGVARBINARY" + "SQL_LONGVARNCHAR" + "SQL_LONGVARCHAR" + "SQL_NCHAR" + "SQL_NCLOB" + "SQL_NUMERIC" + "SQL_NVARCHAR" + "SQL_REAL" + "SQL_SMALLINT" + "SQL_TIME" + "SQL_TIMESTAMP" + "SQL_TINYINT" + "SQL_TSI_DAY" + "SQL_TSI_FRAC_SECOND" + "SQL_TSI_HOUR" + "SQL_TSI_MICROSECOND" + "SQL_TSI_MINUTE" + "SQL_TSI_MONTH" + "SQL_TSI_QUARTER" + "SQL_TSI_SECOND" + "SQL_TSI_WEEK" + "SQL_TSI_YEAR" + "SQL_VARBINARY" + "SQL_VARCHAR" + "STATE" + "STATEMENT" + "STRUCTURE" + "STYLE" + "SUBCLASS_ORIGIN" + "SUBSTITUTE" + "TABLE_NAME" + "TEMPORARY" + "TIES" + "TIMESTAMPADD" + "TIMESTAMPDIFF" + "TOP_LEVEL_COUNT" + "TRANSACTION" + "TRANSACTIONS_ACTIVE" + "TRANSACTIONS_COMMITTED" + "TRANSACTIONS_ROLLED_BACK" + "TRANSFORM" + "TRANSFORMS" + "TRIGGER_CATALOG" + "TRIGGER_NAME" + "TRIGGER_SCHEMA" + "TYPE" + "UNBOUNDED" + "UNCOMMITTED" + "UNCONDITIONAL" + "UNDER" + "UNNAMED" + "USAGE" + "USER_DEFINED_TYPE_CATALOG" + "USER_DEFINED_TYPE_CODE" + "USER_DEFINED_TYPE_NAME" + "USER_DEFINED_TYPE_SCHEMA" + "UTF8" + "UTF16" + "UTF32" + "VERSION" + "VIEW" + "WEEK" + "WRAPPER" + "WORK" + "WRITE" + "XML" + "ZONE" + ] + + # List of additional join types. Each is a method with no arguments. + # Example: LeftSemiJoin() + joinTypes: [ + ] + + # List of methods for parsing custom SQL statements. + # Return type of method implementation should be 'SqlNode'. + # Example: SqlShowDatabases(), SqlShowTables(). + statementParserMethods: [ + "DruidSqlInsert()" + ] + + # List of methods for parsing custom literals. + # Return type of method implementation should be "SqlNode". + # Example: ParseJsonLiteral(). + literalParserMethods: [ + ] + + # List of methods for parsing custom data types. + # Return type of method implementation should be "SqlTypeNameSpec". + # Example: SqlParseTimeStampZ(). + dataTypeParserMethods: [ + ] + + # List of methods for parsing builtin function calls. + # Return type of method implementation should be "SqlNode". + # Example: DateFunctionCall(). + builtinFunctionCallMethods: [ + ] + + # List of methods for parsing extensions to "ALTER " calls. + # Each must accept arguments "(SqlParserPos pos, String scope)". + # Example: "SqlUploadJarNode" + alterStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls. + # Each must accept arguments "(SqlParserPos pos, boolean replace)". + createStatementParserMethods: [ + ] + + # List of methods for parsing extensions to "DROP" calls. + # Each must accept arguments "(SqlParserPos pos)". + dropStatementParserMethods: [ + ] + + # Binary operators tokens + binaryOperatorsTokens: [ + ] + + # Binary operators initialization + extraBinaryExpressions: [ + ] + + # List of files in @includes directory that have parser method + # implementations for parsing custom SQL statements, literals or types + # given as part of "statementParserMethods", "literalParserMethods" or + # "dataTypeParserMethods". + implementationFiles: [ + "insert.ftl" + ] + + includePosixOperators: false + includeCompoundIdentifier: true + includeBraces: true + includeAdditionalDeclarations: false + } +} + +freemarkerLinks: { + includes: includes/ +} diff --git a/sql/src/main/codegen/includes/insert.ftl b/sql/src/main/codegen/includes/insert.ftl new file mode 100644 index 000000000000..1a0fc1db9d22 --- /dev/null +++ b/sql/src/main/codegen/includes/insert.ftl @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Using fully qualified name for Pair class, since Calcite also has a same class name being used in the Parser.jj +SqlNode DruidSqlInsert() : +{ + SqlNode insertNode; + org.apache.druid.java.util.common.Pair partitionedBy = null; + SqlNodeList clusteredBy = null; +} +{ + insertNode = SqlInsert() + + partitionedBy = PartitionGranularity() + [ + + clusteredBy = ClusterItems() + ] + { + if (!(insertNode instanceof SqlInsert)) { + // This shouldn't be encountered, but done as a defensive practice. SqlInsert() always returns a node of type + // SqlInsert + return insertNode; + } + SqlInsert sqlInsert = (SqlInsert) insertNode; + return new DruidSqlInsert(sqlInsert, partitionedBy.lhs, partitionedBy.rhs, clusteredBy); + } +} + +SqlNodeList ClusterItems() : +{ + List list; + final Span s; + SqlNode e; +} +{ + e = OrderItem() { + s = span(); + list = startList(e); + } + ( + LOOKAHEAD(2) e = OrderItem() { list.add(e); } + )* + { + return new SqlNodeList(list, s.addAll(list).pos()); + } +} + +org.apache.druid.java.util.common.Pair PartitionGranularity() : +{ + SqlNode e = null; + org.apache.druid.java.util.common.granularity.Granularity granularity = null; + String unparseString = null; +} +{ + ( + + { + granularity = Granularities.HOUR; + unparseString = "HOUR"; + } + | + + { + granularity = Granularities.DAY; + unparseString = "DAY"; + } + | + + { + granularity = Granularities.MONTH; + unparseString = "MONTH"; + } + | + + { + granularity = Granularities.YEAR; + unparseString = "YEAR"; + } + | +