org.apache.flink.table.planner.delegation.hive.copy

Class HiveParserBaseSemanticAnalyzer

java.lang.Object

org.apache.flink.table.planner.delegation.hive.copy.HiveParserBaseSemanticAnalyzer

public class HiveParserBaseSemanticAnalyzer
extends Object

Counterpart of hive's org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer, and also contains code from SemanticAnalyzer and CalcitePlanner in order to limit file sizes.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	HiveParserBaseSemanticAnalyzer.AggInfo Counterpart of hive's CalcitePlanner.AggInfo.
static class	HiveParserBaseSemanticAnalyzer.AnalyzeRewriteContext Counterpart of hive's BaseSemanticAnalyzer.AnalyzeRewriteContext.
static class	HiveParserBaseSemanticAnalyzer.GenericUDAFInfo Counterpart of hive's SemanticAnalyzer.GenericUDAFInfo.
static class	HiveParserBaseSemanticAnalyzer.HiveParserRowFormatParams Counterpart of hive's BaseSemanticAnalyzer.RowFormatParams.
static class	HiveParserBaseSemanticAnalyzer.NotNullConstraint Counterpart of hive's SQLNotNullConstraint.
static class	HiveParserBaseSemanticAnalyzer.Phase1Ctx Counterpart of hive's SemanticAnalyzer.Phase1Ctx.
static class	HiveParserBaseSemanticAnalyzer.PrimaryKey Counterpart of hive's SQLPrimaryKey.
static class	HiveParserBaseSemanticAnalyzer.SerDeClassProps including serde class name and the properties.
static class	HiveParserBaseSemanticAnalyzer.TableSpec Counterpart of hive's BaseSemanticAnalyzer.TableSpec.
static class	HiveParserBaseSemanticAnalyzer.TableType Counterpart of hive's CalcitePlanner.TableType.

Method Summary

Modifier and Type	Method and Description
static void	addToGBExpr(HiveParserRowResolver groupByOutputRowResolver, HiveParserRowResolver groupByInputRowResolver, HiveParserASTNode grpbyExpr, org.apache.hadoop.hive.ql.plan.ExprNodeDesc grpbyExprNDesc, List<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> gbExprNDescLst, List<String> outputColumnNames)
static Map<String,Integer>	buildHiveColNameToInputPosMap(List<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> colList, HiveParserRowResolver inputRR)
static Map<String,Integer>	buildHiveToCalciteColumnMap(HiveParserRowResolver rr)
static Tuple2<String,String>	charSetString(String charSetName, String charSetString)
static org.apache.calcite.util.ImmutableBitSet	convert(int value, int length)
static org.apache.calcite.rel.RelNode	genValues(String tabAlias, ResolvedCatalogTable catalogTable, HiveParserRowResolver rowResolver, org.apache.calcite.plan.RelOptCluster cluster, List<List<String>> values)
static org.apache.calcite.rex.RexWindowBound	getBound(HiveParserWindowingSpec.BoundarySpec spec, org.apache.calcite.plan.RelOptCluster cluster)
static String	getColumnInternalName(int pos)
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	getColumns(HiveParserASTNode ast)
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	getColumns(HiveParserASTNode ast, boolean lowerCase)
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	getColumns(HiveParserASTNode ast, boolean lowerCase, List<HiveParserBaseSemanticAnalyzer.PrimaryKey> primaryKeys, List<HiveParserBaseSemanticAnalyzer.NotNullConstraint> notNulls)
static org.apache.calcite.util.Pair<List<org.apache.calcite.rel.core.CorrelationId>,org.apache.calcite.util.ImmutableBitSet>	getCorrelationUse(org.apache.calcite.rex.RexCall call)
static Map.Entry<String,String>	getDbTableNamePair(HiveParserASTNode tableNameNode)
static String	getDotName(String... names)
static List<HiveParserASTNode>	getGroupByForClause(HiveParserQBParseInfo parseInfo, String dest)
static List<Integer>	getGroupingSets(List<HiveParserASTNode> groupByExpr, HiveParserQBParseInfo parseInfo, String dest)
static List<Integer>	getGroupingSetsForCube(int size)
static List<Integer>	getGroupingSetsForRollup(int size)
static HiveParserBaseSemanticAnalyzer.AggInfo	getHiveAggInfo(HiveParserASTNode aggAst, int aggFnLstArgIndx, HiveParserRowResolver inputRR, HiveParserWindowingSpec.WindowFunctionSpec winFuncSpec, HiveParserSemanticAnalyzer semanticAnalyzer, org.apache.calcite.tools.FrameworkConfig frameworkConfig, org.apache.calcite.plan.RelOptCluster cluster)
static ObjectIdentifier	getObjectIdentifier(CatalogRegistry catalogRegistry, HiveParserASTNode tabNameNode)
static List<org.apache.calcite.rex.RexFieldCollation>	getOrderKeys(HiveParserPTFInvocationSpec.OrderSpec orderSpec, HiveParserRexNodeConverter converter, HiveParserRowResolver inputRR, HiveParserTypeCheckCtx typeCheckCtx, HiveParserSemanticAnalyzer semanticAnalyzer)
static List<org.apache.calcite.rex.RexNode>	getPartitionKeys(HiveParserPTFInvocationSpec.PartitionSpec partitionSpec, HiveParserRexNodeConverter converter, HiveParserRowResolver inputRR, HiveParserTypeCheckCtx typeCheckCtx, HiveParserSemanticAnalyzer semanticAnalyzer)
static UnresolvedIdentifier	getQualifiedTableName(HiveParserASTNode tabNameNode)
static ResolvedCatalogBaseTable<?>	getResolvedCatalogBaseTable(CatalogRegistry catalogRegistry, ObjectIdentifier tableIdentifier)
static String	getTypeStringFromAST(HiveParserASTNode typeNode)
static String	getUnescapedName(HiveParserASTNode tableOrColumnNode) Get dequoted name from a table/column node.
static String	getUnescapedName(HiveParserASTNode tableOrColumnNode, String currentDatabase)
static String	getUnescapedName(HiveParserASTNode tableOrColumnNode, String currentCatalog, String currentDatabase)
static String	getUnescapedOriginTableName(HiveParserASTNode node) Get the unescaped origin table name for the table node.
static String	getUnescapedUnqualifiedTableName(HiveParserASTNode node) Get the unqualified name from a table node.
static Set<org.apache.calcite.rel.core.CorrelationId>	getVariablesSetForFilter(org.apache.calcite.rex.RexNode rexNode) traverse the given node to find all correlated variables, the main logic is from HiveFilter.getVariablesSet().
static int	getWindowSpecIndx(HiveParserASTNode wndAST)
static HiveParserBaseSemanticAnalyzer.Phase1Ctx	initPhase1Ctx()
static HiveParserBaseSemanticAnalyzer.TableType	obtainTableType(org.apache.hadoop.hive.ql.metadata.Table tabMetaData)
static ObjectIdentifier	parseCompoundName(CatalogRegistry catalogRegistry, String compoundName)
static void	processPositionAlias(HiveParserASTNode ast, HiveConf conf)
static void	readProps(HiveParserASTNode prop, Map<String,String> mapProp) Converts parsed key/value properties pairs into a map.
static void	removeOBInSubQuery(HiveParserQBExpr qbExpr)
static String	stripQuotes(String val)
static boolean	topLevelConjunctCheck(HiveParserASTNode searchCond, org.apache.hadoop.hive.common.ObjectPair<Boolean,Integer> subqInfo)
static String	unescapeIdentifier(String val) Remove the encapsulating "`" pair from the identifier.
static String	unescapeSQLString(String b)
static void	validateNoHavingReferenceToAlias(HiveParserQB qb, HiveParserASTNode havingExpr, HiveParserRowResolver inputRR, HiveParserSemanticAnalyzer semanticAnalyzer)
static void	validatePartColumnType(ResolvedCatalogTable resolvedCatalogTable, Map<String,String> partSpec, HiveParserASTNode astNode, HiveConf conf, org.apache.calcite.tools.FrameworkConfig frameworkConfig, org.apache.calcite.plan.RelOptCluster cluster)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

getColumns

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> getColumns(HiveParserASTNode ast)
                                                                         throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getColumns

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> getColumns(HiveParserASTNode ast,
                                                                                boolean lowerCase)
                                                                         throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getTypeStringFromAST

public static String getTypeStringFromAST(HiveParserASTNode typeNode)
                                   throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getColumns

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> getColumns(HiveParserASTNode ast,
                                                                                boolean lowerCase,
                                                                                List<HiveParserBaseSemanticAnalyzer.PrimaryKey> primaryKeys,
                                                                                List<HiveParserBaseSemanticAnalyzer.NotNullConstraint> notNulls)
                                                                         throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getDotName

public static String getDotName(String... names)

readProps

public static void readProps(HiveParserASTNode prop,
                             Map<String,String> mapProp)

Converts parsed key/value properties pairs into a map.

Parameters:

prop - HiveParserASTNode parent of the key/value pairs

mapProp - property map which receives the mappings

getObjectIdentifier

public static ObjectIdentifier getObjectIdentifier(CatalogRegistry catalogRegistry,
                                                   HiveParserASTNode tabNameNode)
                                            throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

parseCompoundName

public static ObjectIdentifier parseCompoundName(CatalogRegistry catalogRegistry,
                                                 String compoundName)

getQualifiedTableName

public static UnresolvedIdentifier getQualifiedTableName(HiveParserASTNode tabNameNode)
                                                  throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

charSetString

public static Tuple2<String,String> charSetString(String charSetName,
                                                  String charSetString)
                                           throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

stripQuotes

public static String stripQuotes(String val)

unescapeIdentifier

public static String unescapeIdentifier(String val)

Remove the encapsulating "`" pair from the identifier. We allow users to use "`" to escape identifier for table names, column names and aliases, in case that coincide with Hive language keywords.

getUnescapedUnqualifiedTableName

public static String getUnescapedUnqualifiedTableName(HiveParserASTNode node)
                                               throws org.apache.hadoop.hive.ql.parse.SemanticException

Get the unqualified name from a table node. This method works for table names qualified with their schema (e.g., "catalog.db.table") and table names without schema qualification. In both cases, it returns the table name without the schema.

Parameters:

node - the table node

Returns:

the table name without schema qualification (i.e., if name is "catalog.db.table" or "table", returns "table")

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getUnescapedName

public static String getUnescapedName(HiveParserASTNode tableOrColumnNode)
                               throws org.apache.hadoop.hive.ql.parse.SemanticException

Get dequoted name from a table/column node.

Parameters:

tableOrColumnNode - the table or column node

Returns:

for table node, return the table that users specific like catalog.db.tab, db.tab or tab. For column node column, return col.

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getUnescapedName

public static String getUnescapedName(HiveParserASTNode tableOrColumnNode,
                                      @Nullable
                                      String currentCatalog,
                                      @Nullable
                                      String currentDatabase)
                               throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getUnescapedOriginTableName

public static String getUnescapedOriginTableName(HiveParserASTNode node)
                                          throws org.apache.hadoop.hive.ql.parse.SemanticException

Get the unescaped origin table name for the table node. This method returns "catalog.db.table","db.table" or "table" according to what the table node actually specifies

Parameters:

node - the table node

Returns:

"catalog.db.table", "db.table" or "table"

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getUnescapedName

public static String getUnescapedName(HiveParserASTNode tableOrColumnNode,
                                      String currentDatabase)

getDbTableNamePair

public static Map.Entry<String,String> getDbTableNamePair(HiveParserASTNode tableNameNode)

unescapeSQLString

public static String unescapeSQLString(String b)

getColumnInternalName

public static String getColumnInternalName(int pos)

getGroupingSetsForRollup

public static List<Integer> getGroupingSetsForRollup(int size)

getGroupingSetsForCube

public static List<Integer> getGroupingSetsForCube(int size)

getGroupingSets

public static List<Integer> getGroupingSets(List<HiveParserASTNode> groupByExpr,
                                            HiveParserQBParseInfo parseInfo,
                                            String dest)
                                     throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getGroupByForClause

public static List<HiveParserASTNode> getGroupByForClause(HiveParserQBParseInfo parseInfo,
                                                          String dest)

getBound

public static org.apache.calcite.rex.RexWindowBound getBound(HiveParserWindowingSpec.BoundarySpec spec,
                                                             org.apache.calcite.plan.RelOptCluster cluster)

initPhase1Ctx

public static HiveParserBaseSemanticAnalyzer.Phase1Ctx initPhase1Ctx()

processPositionAlias

public static void processPositionAlias(HiveParserASTNode ast,
                                        HiveConf conf)
                                 throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

removeOBInSubQuery

public static void removeOBInSubQuery(HiveParserQBExpr qbExpr)

obtainTableType

public static HiveParserBaseSemanticAnalyzer.TableType obtainTableType(org.apache.hadoop.hive.ql.metadata.Table tabMetaData)

convert

public static org.apache.calcite.util.ImmutableBitSet convert(int value,
                                                              int length)

buildHiveColNameToInputPosMap

public static Map<String,Integer> buildHiveColNameToInputPosMap(List<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> colList,
                                                                HiveParserRowResolver inputRR)

buildHiveToCalciteColumnMap

public static Map<String,Integer> buildHiveToCalciteColumnMap(HiveParserRowResolver rr)

getCorrelationUse

public static org.apache.calcite.util.Pair<List<org.apache.calcite.rel.core.CorrelationId>,org.apache.calcite.util.ImmutableBitSet> getCorrelationUse(org.apache.calcite.rex.RexCall call)

topLevelConjunctCheck

public static boolean topLevelConjunctCheck(HiveParserASTNode searchCond,
                                            org.apache.hadoop.hive.common.ObjectPair<Boolean,Integer> subqInfo)

addToGBExpr

public static void addToGBExpr(HiveParserRowResolver groupByOutputRowResolver,
                               HiveParserRowResolver groupByInputRowResolver,
                               HiveParserASTNode grpbyExpr,
                               org.apache.hadoop.hive.ql.plan.ExprNodeDesc grpbyExprNDesc,
                               List<org.apache.hadoop.hive.ql.plan.ExprNodeDesc> gbExprNDescLst,
                               List<String> outputColumnNames)

getWindowSpecIndx

public static int getWindowSpecIndx(HiveParserASTNode wndAST)

validateNoHavingReferenceToAlias

public static void validateNoHavingReferenceToAlias(HiveParserQB qb,
                                                    HiveParserASTNode havingExpr,
                                                    HiveParserRowResolver inputRR,
                                                    HiveParserSemanticAnalyzer semanticAnalyzer)
                                             throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getPartitionKeys

public static List<org.apache.calcite.rex.RexNode> getPartitionKeys(HiveParserPTFInvocationSpec.PartitionSpec partitionSpec,
                                                                    HiveParserRexNodeConverter converter,
                                                                    HiveParserRowResolver inputRR,
                                                                    HiveParserTypeCheckCtx typeCheckCtx,
                                                                    HiveParserSemanticAnalyzer semanticAnalyzer)
                                                             throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getOrderKeys

public static List<org.apache.calcite.rex.RexFieldCollation> getOrderKeys(HiveParserPTFInvocationSpec.OrderSpec orderSpec,
                                                                          HiveParserRexNodeConverter converter,
                                                                          HiveParserRowResolver inputRR,
                                                                          HiveParserTypeCheckCtx typeCheckCtx,
                                                                          HiveParserSemanticAnalyzer semanticAnalyzer)
                                                                   throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getHiveAggInfo

public static HiveParserBaseSemanticAnalyzer.AggInfo getHiveAggInfo(HiveParserASTNode aggAst,
                                                                    int aggFnLstArgIndx,
                                                                    HiveParserRowResolver inputRR,
                                                                    HiveParserWindowingSpec.WindowFunctionSpec winFuncSpec,
                                                                    HiveParserSemanticAnalyzer semanticAnalyzer,
                                                                    org.apache.calcite.tools.FrameworkConfig frameworkConfig,
                                                                    org.apache.calcite.plan.RelOptCluster cluster)
                                                             throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

genValues

public static org.apache.calcite.rel.RelNode genValues(String tabAlias,
                                                       ResolvedCatalogTable catalogTable,
                                                       HiveParserRowResolver rowResolver,
                                                       org.apache.calcite.plan.RelOptCluster cluster,
                                                       List<List<String>> values)

getVariablesSetForFilter

public static Set<org.apache.calcite.rel.core.CorrelationId> getVariablesSetForFilter(org.apache.calcite.rex.RexNode rexNode)

traverse the given node to find all correlated variables, the main logic is from HiveFilter.getVariablesSet().

validatePartColumnType

public static void validatePartColumnType(ResolvedCatalogTable resolvedCatalogTable,
                                          Map<String,String> partSpec,
                                          HiveParserASTNode astNode,
                                          HiveConf conf,
                                          org.apache.calcite.tools.FrameworkConfig frameworkConfig,
                                          org.apache.calcite.plan.RelOptCluster cluster)
                                   throws org.apache.hadoop.hive.ql.parse.SemanticException

Throws:

org.apache.hadoop.hive.ql.parse.SemanticException

getResolvedCatalogBaseTable

public static ResolvedCatalogBaseTable<?> getResolvedCatalogBaseTable(CatalogRegistry catalogRegistry,
                                                                      ObjectIdentifier tableIdentifier)