org.apache.flink.table.planner.delegation.hive

Class HiveParserUtils

java.lang.Object

org.apache.flink.table.planner.delegation.hive.HiveParserUtils

public class HiveParserUtils
extends Object

Util class for the hive parser.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	HiveParserUtils.CorrelationCollector A visitor to collect correlation IDs and required columns.
static class	HiveParserUtils.LateralViewInfo Information needed to generate logical plan for a lateral view.

Field Summary

Fields

Modifier and Type	Field and Description
static String	BRIDGING_SQL_FUNCTION_CLZ_NAME

Method Summary

Modifier and Type	Method and Description
static org.apache.hadoop.hive.ql.hooks.ReadEntity	addInput(Set<org.apache.hadoop.hive.ql.hooks.ReadEntity> inputs, org.apache.hadoop.hive.ql.hooks.ReadEntity newInput, boolean mergeIsDirectFlag)
static org.apache.calcite.util.NlsString	asUnicodeString(String text)
static String	canHandleQbForCbo(org.apache.hadoop.hive.ql.QueryProperties queryProperties)
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	convertRowSchemaToResultSetSchema(HiveParserRowResolver rr, boolean useTabAliasIfAvailable)
static void	extractColumns(Set<String> colNamesExprs, org.apache.hadoop.hive.ql.plan.ExprNodeDesc exprNode)
static HiveParserUtils.LateralViewInfo	extractLateralViewInfo(HiveParserASTNode lateralView, HiveParserRowResolver inputRR, HiveParserSemanticAnalyzer hiveAnalyzer, org.apache.calcite.tools.FrameworkConfig frameworkConfig, org.apache.calcite.plan.RelOptCluster cluster)
static String	generateErrorMessage(HiveParserASTNode ast, String message)
static Map<HiveParserASTNode,org.apache.hadoop.hive.ql.plan.ExprNodeDesc>	genExprNode(HiveParserASTNode expr, HiveParserTypeCheckCtx tcCtx)
static org.apache.calcite.rel.RelNode	genFilterRelNode(org.apache.calcite.rel.RelNode relNode, org.apache.calcite.rex.RexNode rexNode, Collection<org.apache.calcite.rel.core.CorrelationId> variables)
static HiveParserASTNode	genSelectDIAST(HiveParserRowResolver rr)
static org.apache.calcite.rel.RelNode	genValuesRelNode(org.apache.calcite.plan.RelOptCluster cluster, org.apache.calcite.rel.type.RelDataType rowType, List<List<org.apache.calcite.rex.RexLiteral>> rows)
static org.apache.calcite.sql.SqlOperator	getAnySqlOperator(String funcName, org.apache.calcite.sql.SqlOperatorTable opTable)
static String[]	getColAlias(HiveParserASTNode selExpr, String defaultName, HiveParserRowResolver inputRR, boolean includeFuncName, int colNum)
static org.apache.hadoop.hive.ql.exec.FunctionInfo	getFunctionInfo(String funcName)
static org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator	getGenericUDAFEvaluator(String aggName, ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> aggParameters, HiveParserASTNode aggTree, boolean isDistinct, boolean isAllColumns, org.apache.calcite.sql.SqlOperatorTable opTable)
static HiveParserBaseSemanticAnalyzer.GenericUDAFInfo	getGenericUDAFInfo(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator evaluator, org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode emode, ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> aggParameters) Returns the GenericUDAFInfo struct for the aggregation.
static List<org.apache.calcite.rex.RexNode>	getProjsFromBelowAsInputRef(org.apache.calcite.rel.RelNode rel)
static HiveShim	getSessionHiveShim()
static org.apache.calcite.sql.SqlOperator	getSqlOperator(String funcName, org.apache.calcite.sql.SqlOperatorTable opTable, org.apache.calcite.sql.SqlFunctionCategory category)
static String	getStandardDisplayString(String name, String[] children)
static ArrayList<org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector>	getWritableObjectInspector(ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> exprs) Convert exprNodeDesc array to ObjectInspector array.
static org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode	groupByDescModeToUDAFMode(org.apache.hadoop.hive.ql.plan.GroupByDesc.Mode mode, boolean isDistinct)
static boolean	hasCommonElement(Set<String> set1, Set<String> set2)
static org.apache.calcite.rel.type.RelDataType	inferReturnTypeForOperands(org.apache.calcite.sql.SqlOperator sqlOperator, List<org.apache.calcite.rex.RexNode> operands, org.apache.calcite.rel.type.RelDataTypeFactory dataTypeFactory)
static org.apache.calcite.rel.type.RelDataType	inferReturnTypeForOperandsTypes(org.apache.calcite.sql.SqlOperator sqlOperator, List<org.apache.calcite.rel.type.RelDataType> types, List<org.apache.calcite.rex.RexNode> operands, org.apache.calcite.rel.type.RelDataTypeFactory dataTypeFactory)
static boolean	isBridgingSqlFunction(org.apache.calcite.sql.SqlOperator sqlOperator)
static boolean	isFromTimeStampToDecimal(org.apache.calcite.rel.type.RelDataType srcType, org.apache.calcite.rel.type.RelDataType targetType)
static boolean	isIdentityProject(org.apache.calcite.rel.RelNode input, List<org.apache.calcite.rex.RexNode> exprs, List<String> aliases)
static boolean	isJoinToken(HiveParserASTNode node)
static boolean	isNative(org.apache.calcite.sql.SqlOperator sqlOperator)
static boolean	isOuterJoinToken(HiveParserASTNode node)
static boolean	isRegex(String pattern, HiveConf conf) Returns whether the pattern is a regex expression (instead of a normal string).
static boolean	isSkewedCol(String alias, HiveParserQB qb, String colName)
static boolean	isUDAF(org.apache.calcite.sql.SqlOperator sqlOperator)
static boolean	isUDTF(org.apache.calcite.sql.SqlOperator sqlOperator)
static boolean	isValuesTempTable(String tableName) Check if the table is the temporary table created by VALUES() syntax.
static boolean	legacyGrouping(org.apache.hadoop.conf.Configuration conf)
static org.apache.calcite.rex.RexNode	makeOver(org.apache.calcite.rex.RexBuilder rexBuilder, org.apache.calcite.rel.type.RelDataType type, org.apache.calcite.sql.SqlAggFunction operator, List<org.apache.calcite.rex.RexNode> exprs, List<org.apache.calcite.rex.RexNode> partitionKeys, List<org.apache.calcite.rex.RexFieldCollation> orderKeys, org.apache.calcite.rex.RexWindowBound lowerBound, org.apache.calcite.rex.RexWindowBound upperBound, boolean physical, boolean allowPartial, boolean nullWhenCountZero, boolean distinct, boolean ignoreNulls) Proxy to RexBuilder.makeOver(RelDataType, SqlAggFunction, List, List, com.google.common.collect.ImmutableList, RexWindowBound, RexWindowBound, boolean, boolean, boolean, boolean, boolean).
static boolean	pivotResult(String functionName)
static org.apache.calcite.rex.RexNode	projectNonColumnEquiConditions(org.apache.calcite.rel.core.RelFactories.ProjectFactory factory, org.apache.calcite.rel.RelNode[] inputRels, List<org.apache.calcite.rex.RexNode> leftJoinKeys, List<org.apache.calcite.rex.RexNode> rightJoinKeys, int systemColCount, List<Integer> leftKeys, List<Integer> rightKeys) Push any equi join conditions that are not column references as Projections on top of the children.
static void	removeASTChild(HiveParserASTNode node)
static HiveParserASTNode	rewriteGroupingFunctionAST(List<HiveParserASTNode> grpByAstExprs, HiveParserASTNode targetNode, boolean noneSet)
static org.apache.calcite.rex.RexSubQuery	rexSubQueryIn(org.apache.calcite.rel.RelNode relNode, Collection<org.apache.calcite.rex.RexNode> rexNodes) Proxy to RexSubQuery.in(RelNode, com.google.common.collect.ImmutableList).
static void	saveViewDefinition(List<org.apache.hadoop.hive.metastore.api.FieldSchema> resultSchema, HiveParserCreateViewInfo createViewInfo, org.antlr.runtime.TokenRewriteStream tokenRewriteStream, HiveParserUnparseTranslator unparseTranslator, HiveConf conf)
static org.apache.calcite.rel.core.AggregateCall	toAggCall(HiveParserBaseSemanticAnalyzer.AggInfo aggInfo, HiveParserRexNodeConverter converter, Map<String,Integer> rexNodeToPos, int groupCount, org.apache.calcite.rel.RelNode input, org.apache.calcite.plan.RelOptCluster cluster, SqlFunctionConverter funcConverter)
static DataType	toDataType(org.apache.calcite.rel.type.RelDataType relDataType)
static org.apache.calcite.rel.type.RelDataType	toRelDataType(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo, org.apache.calcite.rel.type.RelDataTypeFactory relTypeFactory)
static int	unsetBit(int bitmap, int bitIdx)
static void	verifyCanHandleAst(HiveParserASTNode ast, HiveParserQB qb, org.apache.hadoop.hive.ql.QueryProperties queryProperties)
static void	writeAsText(String text, org.apache.hadoop.fs.FSDataOutputStream out) Convert a string to Text format and write its bytes in the same way TextOutputFormat would do.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

BRIDGING_SQL_FUNCTION_CLZ_NAME

public static final String BRIDGING_SQL_FUNCTION_CLZ_NAME

See Also:

Constant Field Values

Method Detail

removeASTChild

public static void removeASTChild(HiveParserASTNode node)

asUnicodeString

public static org.apache.calcite.util.NlsString asUnicodeString(String text)

canHandleQbForCbo

public static String canHandleQbForCbo(org.apache.hadoop.hive.ql.QueryProperties queryProperties)

toRelDataType

public static org.apache.calcite.rel.type.RelDataType toRelDataType(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo typeInfo,
                                                                    org.apache.calcite.rel.type.RelDataTypeFactory relTypeFactory)
                                                             throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

makeOver

public static org.apache.calcite.rex.RexNode makeOver(org.apache.calcite.rex.RexBuilder rexBuilder,
                                                      org.apache.calcite.rel.type.RelDataType type,
                                                      org.apache.calcite.sql.SqlAggFunction operator,
                                                      List<org.apache.calcite.rex.RexNode> exprs,
                                                      List<org.apache.calcite.rex.RexNode> partitionKeys,
                                                      List<org.apache.calcite.rex.RexFieldCollation> orderKeys,
                                                      org.apache.calcite.rex.RexWindowBound lowerBound,
                                                      org.apache.calcite.rex.RexWindowBound upperBound,
                                                      boolean physical,
                                                      boolean allowPartial,
                                                      boolean nullWhenCountZero,
                                                      boolean distinct,
                                                      boolean ignoreNulls)

Proxy to RexBuilder.makeOver(RelDataType, SqlAggFunction, List, List, com.google.common.collect.ImmutableList, RexWindowBound, RexWindowBound, boolean, boolean, boolean, boolean, boolean).

genValuesRelNode

public static org.apache.calcite.rel.RelNode genValuesRelNode(org.apache.calcite.plan.RelOptCluster cluster,
                                                              org.apache.calcite.rel.type.RelDataType rowType,
                                                              List<List<org.apache.calcite.rex.RexLiteral>> rows)

genFilterRelNode

public static org.apache.calcite.rel.RelNode genFilterRelNode(org.apache.calcite.rel.RelNode relNode,
                                                              org.apache.calcite.rex.RexNode rexNode,
                                                              Collection<org.apache.calcite.rel.core.CorrelationId> variables)

rexSubQueryIn

public static org.apache.calcite.rex.RexSubQuery rexSubQueryIn(org.apache.calcite.rel.RelNode relNode,
                                                               Collection<org.apache.calcite.rex.RexNode> rexNodes)

Proxy to RexSubQuery.in(RelNode, com.google.common.collect.ImmutableList).

isValuesTempTable

public static boolean isValuesTempTable(String tableName)

Check if the table is the temporary table created by VALUES() syntax.

Parameters:

tableName - table name

addInput

public static org.apache.hadoop.hive.ql.hooks.ReadEntity addInput(Set<org.apache.hadoop.hive.ql.hooks.ReadEntity> inputs,
                                                                  org.apache.hadoop.hive.ql.hooks.ReadEntity newInput,
                                                                  boolean mergeIsDirectFlag)

genExprNode

public static Map<HiveParserASTNode,org.apache.hadoop.hive.ql.plan.ExprNodeDesc> genExprNode(HiveParserASTNode expr,
                                                                                             HiveParserTypeCheckCtx tcCtx)
                                                                                      throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

generateErrorMessage

public static String generateErrorMessage(HiveParserASTNode ast,
                                          String message)

writeAsText

public static void writeAsText(String text,
                               org.apache.hadoop.fs.FSDataOutputStream out)
                        throws IOException

Convert a string to Text format and write its bytes in the same way TextOutputFormat would do. This is needed to properly encode non-ascii characters.

Throws:

IOException

genSelectDIAST

public static HiveParserASTNode genSelectDIAST(HiveParserRowResolver rr)

groupByDescModeToUDAFMode

public static org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode groupByDescModeToUDAFMode(org.apache.hadoop.hive.ql.plan.GroupByDesc.Mode mode,
                                                                                                        boolean isDistinct)

isSkewedCol

public static boolean isSkewedCol(String alias,
                                  HiveParserQB qb,
                                  String colName)

isJoinToken

public static boolean isJoinToken(HiveParserASTNode node)

isOuterJoinToken

public static boolean isOuterJoinToken(HiveParserASTNode node)

extractColumns

public static void extractColumns(Set<String> colNamesExprs,
                                  org.apache.hadoop.hive.ql.plan.ExprNodeDesc exprNode)

hasCommonElement

public static boolean hasCommonElement(Set<String> set1,
                                       Set<String> set2)

getGenericUDAFInfo

public static HiveParserBaseSemanticAnalyzer.GenericUDAFInfo getGenericUDAFInfo(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator evaluator,
                                                                                org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode emode,
                                                                                ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> aggParameters)
                                                                         throws org.apache.hadoop.hive.ql.parse.SemanticException

Returns the GenericUDAFInfo struct for the aggregation.

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getWritableObjectInspector

public static ArrayList<org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector> getWritableObjectInspector(ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> exprs)

Convert exprNodeDesc array to ObjectInspector array.

getGenericUDAFEvaluator

public static org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName,
                                                                                                 ArrayList<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> aggParameters,
                                                                                                 HiveParserASTNode aggTree,
                                                                                                 boolean isDistinct,
                                                                                                 boolean isAllColumns,
                                                                                                 org.apache.calcite.sql.SqlOperatorTable opTable)
                                                                                          throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

isBridgingSqlFunction

public static boolean isBridgingSqlFunction(org.apache.calcite.sql.SqlOperator sqlOperator)

isRegex

public static boolean isRegex(String pattern,
                              HiveConf conf)

Returns whether the pattern is a regex expression (instead of a normal string). Normal string is a string with all alphabets/digits and "_".

getColAlias

public static String[] getColAlias(HiveParserASTNode selExpr,
                                   String defaultName,
                                   HiveParserRowResolver inputRR,
                                   boolean includeFuncName,
                                   int colNum)

unsetBit

public static int unsetBit(int bitmap,
                           int bitIdx)

rewriteGroupingFunctionAST

public static HiveParserASTNode rewriteGroupingFunctionAST(List<HiveParserASTNode> grpByAstExprs,
                                                           HiveParserASTNode targetNode,
                                                           boolean noneSet)

legacyGrouping

public static boolean legacyGrouping(org.apache.hadoop.conf.Configuration conf)

getAnySqlOperator

public static org.apache.calcite.sql.SqlOperator getAnySqlOperator(String funcName,
                                                                   org.apache.calcite.sql.SqlOperatorTable opTable)

getSqlOperator

public static org.apache.calcite.sql.SqlOperator getSqlOperator(String funcName,
                                                                org.apache.calcite.sql.SqlOperatorTable opTable,
                                                                org.apache.calcite.sql.SqlFunctionCategory category)

inferReturnTypeForOperandsTypes

public static org.apache.calcite.rel.type.RelDataType inferReturnTypeForOperandsTypes(org.apache.calcite.sql.SqlOperator sqlOperator,
                                                                                      List<org.apache.calcite.rel.type.RelDataType> types,
                                                                                      List<org.apache.calcite.rex.RexNode> operands,
                                                                                      org.apache.calcite.rel.type.RelDataTypeFactory dataTypeFactory)

inferReturnTypeForOperands

public static org.apache.calcite.rel.type.RelDataType inferReturnTypeForOperands(org.apache.calcite.sql.SqlOperator sqlOperator,
                                                                                 List<org.apache.calcite.rex.RexNode> operands,
                                                                                 org.apache.calcite.rel.type.RelDataTypeFactory dataTypeFactory)

toDataType

public static DataType toDataType(org.apache.calcite.rel.type.RelDataType relDataType)

extractLateralViewInfo

public static HiveParserUtils.LateralViewInfo extractLateralViewInfo(HiveParserASTNode lateralView,
                                                                     HiveParserRowResolver inputRR,
                                                                     HiveParserSemanticAnalyzer hiveAnalyzer,
                                                                     org.apache.calcite.tools.FrameworkConfig frameworkConfig,
                                                                     org.apache.calcite.plan.RelOptCluster cluster)
                                                              throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

isUDAF

public static boolean isUDAF(org.apache.calcite.sql.SqlOperator sqlOperator)

isUDTF

public static boolean isUDTF(org.apache.calcite.sql.SqlOperator sqlOperator)

isNative

public static boolean isNative(org.apache.calcite.sql.SqlOperator sqlOperator)

projectNonColumnEquiConditions

public static org.apache.calcite.rex.RexNode projectNonColumnEquiConditions(org.apache.calcite.rel.core.RelFactories.ProjectFactory factory,
                                                                            org.apache.calcite.rel.RelNode[] inputRels,
                                                                            List<org.apache.calcite.rex.RexNode> leftJoinKeys,
                                                                            List<org.apache.calcite.rex.RexNode> rightJoinKeys,
                                                                            int systemColCount,
                                                                            List<Integer> leftKeys,
                                                                            List<Integer> rightKeys)

Push any equi join conditions that are not column references as Projections on top of the children.

getProjsFromBelowAsInputRef

public static List<org.apache.calcite.rex.RexNode> getProjsFromBelowAsInputRef(org.apache.calcite.rel.RelNode rel)

pivotResult

public static boolean pivotResult(String functionName)
                           throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getFunctionInfo

public static org.apache.hadoop.hive.ql.exec.FunctionInfo getFunctionInfo(String funcName)
                                                                   throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

convertRowSchemaToResultSetSchema

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> convertRowSchemaToResultSetSchema(HiveParserRowResolver rr,
                                                                                                       boolean useTabAliasIfAvailable)

saveViewDefinition

public static void saveViewDefinition(List<org.apache.hadoop.hive.metastore.api.FieldSchema> resultSchema,
                                      HiveParserCreateViewInfo createViewInfo,
                                      org.antlr.runtime.TokenRewriteStream tokenRewriteStream,
                                      HiveParserUnparseTranslator unparseTranslator,
                                      HiveConf conf)
                               throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getSessionHiveShim

public static HiveShim getSessionHiveShim()

getStandardDisplayString

public static String getStandardDisplayString(String name,
                                              String[] children)

verifyCanHandleAst

public static void verifyCanHandleAst(HiveParserASTNode ast,
                                      HiveParserQB qb,
                                      org.apache.hadoop.hive.ql.QueryProperties queryProperties)
                               throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

isIdentityProject

public static boolean isIdentityProject(org.apache.calcite.rel.RelNode input,
                                        List<org.apache.calcite.rex.RexNode> exprs,
                                        List<String> aliases)

toAggCall

public static org.apache.calcite.rel.core.AggregateCall toAggCall(HiveParserBaseSemanticAnalyzer.AggInfo aggInfo,
                                                                  HiveParserRexNodeConverter converter,
                                                                  Map<String,Integer> rexNodeToPos,
                                                                  int groupCount,
                                                                  org.apache.calcite.rel.RelNode input,
                                                                  org.apache.calcite.plan.RelOptCluster cluster,
                                                                  SqlFunctionConverter funcConverter)
                                                           throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

isFromTimeStampToDecimal

public static boolean isFromTimeStampToDecimal(org.apache.calcite.rel.type.RelDataType srcType,
                                               org.apache.calcite.rel.type.RelDataType targetType)