org.apache.flink.table.catalog.hive.util

Class HiveTableUtil

java.lang.Object

org.apache.flink.table.catalog.hive.util.HiveTableUtil

public class HiveTableUtil
extends Object

Utils to for Hive-backed table.

Method Summary

Modifier and Type	Method and Description
static void	alterColumns(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, ResolvedCatalogTable resolvedCatalogTable)
static org.apache.hadoop.hive.metastore.api.Table	alterTableViaCatalogBaseTable(ObjectPath tablePath, ResolvedCatalogBaseTable baseTable, org.apache.hadoop.hive.metastore.api.Table oldHiveTable, HiveConf hiveConf, boolean managedTable)
static void	checkAcidTable(Map<String,String> tableOptions, ObjectPath tablePath) Check whether to read or write on the hive ACID table.
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	createHiveColumns(ResolvedSchema schema) Create Hive columns from Flink ResolvedSchema.
static org.apache.hadoop.hive.metastore.api.Partition	createHivePartition(String dbName, String tableName, List<String> values, org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, Map<String,String> parameters) Creates a Hive partition instance.
static ResolvedSchema	createResolvedSchema(List<org.apache.hadoop.hive.metastore.api.FieldSchema> nonPartCols, List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys, Set<String> notNullColumns, UniqueConstraint primaryKey) Create a Flink's ResolvedSchema from Hive table's columns and partition keys.
static Schema	createSchema(HiveConf hiveConf, org.apache.hadoop.hive.metastore.api.Table hiveTable, HiveMetastoreClientWrapper client, HiveShim hiveShim) Create a Flink's Schema by hive client.
static Schema	createSchema(List<org.apache.hadoop.hive.metastore.api.FieldSchema> nonPartCols, List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys, Set<String> notNullColumns, UniqueConstraint primaryKey) Create a Flink's Schema from Hive table's columns and partition keys.
static byte	enableConstraint(byte trait)
static AlterTableOp	extractAlterTableOp(Map<String,String> props)
static void	extractLocation(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, Map<String,String> properties)
static void	extractRowFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, Map<String,String> properties)
static DataType	extractRowType(HiveConf hiveConf, org.apache.hadoop.hive.metastore.api.Table hiveTable, HiveMetastoreClientWrapper client, HiveShim hiveShim) Create the Hive table's row type.
static void	extractStoredAs(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, Map<String,String> properties, HiveConf hiveConf)
static org.apache.hadoop.conf.Configuration	getHadoopConfiguration(String hadoopConfDir) Returns a new Hadoop Configuration object using the path to the hadoop conf configured.
static List<org.apache.hadoop.hive.metastore.api.FieldSchema>	getNonPartitionFields(HiveConf hiveConf, org.apache.hadoop.hive.metastore.api.Table hiveTable, HiveShim hiveShim)
static org.apache.hadoop.hive.metastore.api.Table	instantiateHiveTable(ObjectPath tablePath, ResolvedCatalogBaseTable table, HiveConf hiveConf, boolean managedTable)
static Optional<String>	makePartitionFilter(int partColOffset, List<String> partColNames, List<Expression> expressions, HiveShim hiveShim) Generates a filter string for partition columns from the given filter expressions.
static byte	relyConstraint(byte trait)
static boolean	requireEnableConstraint(byte trait)
static boolean	requireRelyConstraint(byte trait)
static boolean	requireValidateConstraint(byte trait)
static void	setDefaultStorageFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, HiveConf hiveConf)
static void	setDefaultStorageFormatForDirectory(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, HiveConf hiveConf)
static void	setStorageFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd, String format, HiveConf hiveConf)
static byte	validateConstraint(byte trait)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Method Detail

createSchema

public static Schema createSchema(HiveConf hiveConf,
                                  org.apache.hadoop.hive.metastore.api.Table hiveTable,
                                  HiveMetastoreClientWrapper client,
                                  HiveShim hiveShim)

Create a Flink's Schema by hive client.

createSchema

public static Schema createSchema(List<org.apache.hadoop.hive.metastore.api.FieldSchema> nonPartCols,
                                  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys,
                                  Set<String> notNullColumns,
                                  @Nullable
                                  UniqueConstraint primaryKey)

Create a Flink's Schema from Hive table's columns and partition keys.

createResolvedSchema

public static ResolvedSchema createResolvedSchema(List<org.apache.hadoop.hive.metastore.api.FieldSchema> nonPartCols,
                                                  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys,
                                                  Set<String> notNullColumns,
                                                  @Nullable
                                                  UniqueConstraint primaryKey)

Create a Flink's ResolvedSchema from Hive table's columns and partition keys.

extractRowType

public static DataType extractRowType(HiveConf hiveConf,
                                      org.apache.hadoop.hive.metastore.api.Table hiveTable,
                                      HiveMetastoreClientWrapper client,
                                      HiveShim hiveShim)

Create the Hive table's row type.

createHiveColumns

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> createHiveColumns(ResolvedSchema schema)

Create Hive columns from Flink ResolvedSchema.

createHivePartition

public static org.apache.hadoop.hive.metastore.api.Partition createHivePartition(String dbName,
                                                                                 String tableName,
                                                                                 List<String> values,
                                                                                 org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                                                                 Map<String,String> parameters)

Creates a Hive partition instance.

enableConstraint

public static byte enableConstraint(byte trait)

validateConstraint

public static byte validateConstraint(byte trait)

relyConstraint

public static byte relyConstraint(byte trait)

requireEnableConstraint

public static boolean requireEnableConstraint(byte trait)

requireValidateConstraint

public static boolean requireValidateConstraint(byte trait)

requireRelyConstraint

public static boolean requireRelyConstraint(byte trait)

makePartitionFilter

public static Optional<String> makePartitionFilter(int partColOffset,
                                                   List<String> partColNames,
                                                   List<Expression> expressions,
                                                   HiveShim hiveShim)

Generates a filter string for partition columns from the given filter expressions.

Parameters:

partColOffset - The number of non-partition columns -- used to shift field reference index

partColNames - The names of all partition columns

expressions - The filter expressions in CNF form

Returns:

an Optional filter string equivalent to the expressions, which is empty if the expressions can't be handled

extractLocation

public static void extractLocation(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                   Map<String,String> properties)

extractRowFormat

public static void extractRowFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                    Map<String,String> properties)

extractStoredAs

public static void extractStoredAs(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                   Map<String,String> properties,
                                   HiveConf hiveConf)

setStorageFormat

public static void setStorageFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                    String format,
                                    HiveConf hiveConf)

setDefaultStorageFormat

public static void setDefaultStorageFormat(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                           HiveConf hiveConf)

setDefaultStorageFormatForDirectory

public static void setDefaultStorageFormatForDirectory(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                                       HiveConf hiveConf)

alterColumns

public static void alterColumns(org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
                                ResolvedCatalogTable resolvedCatalogTable)

extractAlterTableOp

public static AlterTableOp extractAlterTableOp(Map<String,String> props)

alterTableViaCatalogBaseTable

public static org.apache.hadoop.hive.metastore.api.Table alterTableViaCatalogBaseTable(ObjectPath tablePath,
                                                                                       ResolvedCatalogBaseTable baseTable,
                                                                                       org.apache.hadoop.hive.metastore.api.Table oldHiveTable,
                                                                                       HiveConf hiveConf,
                                                                                       boolean managedTable)

instantiateHiveTable

public static org.apache.hadoop.hive.metastore.api.Table instantiateHiveTable(ObjectPath tablePath,
                                                                              ResolvedCatalogBaseTable table,
                                                                              HiveConf hiveConf,
                                                                              boolean managedTable)

checkAcidTable

public static void checkAcidTable(Map<String,String> tableOptions,
                                  ObjectPath tablePath)

Check whether to read or write on the hive ACID table.

Parameters:

tableOptions - Hive table options.

tablePath - Identifier table path.

Throws:

FlinkHiveException - Thrown, if the source or sink table is transactional.

getHadoopConfiguration

public static org.apache.hadoop.conf.Configuration getHadoopConfiguration(String hadoopConfDir)

Returns a new Hadoop Configuration object using the path to the hadoop conf configured.

Parameters:

hadoopConfDir - Hadoop conf directory path.

Returns:

A Hadoop configuration instance.

getNonPartitionFields

public static List<org.apache.hadoop.hive.metastore.api.FieldSchema> getNonPartitionFields(HiveConf hiveConf,
                                                                                           org.apache.hadoop.hive.metastore.api.Table hiveTable,
                                                                                           HiveShim hiveShim)