|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Object org.apache.hadoop.hive.ql.stats.StatsUtils
public class StatsUtils
Constructor Summary | |
---|---|
StatsUtils()
|
Method Summary | |
---|---|
static Statistics |
collectStatistics(HiveConf conf,
PrunedPartitionList partList,
Table table,
TableScanOperator tableScanOperator)
Collect table, partition and column level statistics |
static int |
estimateRowSizeFromSchema(HiveConf conf,
List<ColumnInfo> schema,
List<String> neededColumns)
|
static long |
getAvgColLenOfFixedLengthTypes(String colType)
Get size of fixed length primitives |
static long |
getAvgColLenOfVariableLengthTypes(HiveConf conf,
ObjectInspector oi,
String colType)
Get the raw data size of variable length data types |
static List<Long> |
getBasicStatForPartitions(Table table,
List<Partition> parts,
String statType)
Get basic stats of partitions |
static long |
getBasicStatForTable(Table table,
String statType)
Get basic stats of table |
static ColStatistics |
getColStatistics(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj cso,
String tabName,
String colName)
Convert ColumnStatisticsObj to ColStatistics |
static ColStatistics |
getColStatisticsFromExpression(HiveConf conf,
Statistics parentStats,
ExprNodeDesc end)
Get column statistics expression nodes |
static List<ColStatistics> |
getColStatisticsFromExprMap(HiveConf conf,
Statistics parentStats,
Map<String,ExprNodeDesc> colExprMap,
RowSchema rowSchema)
Get column statistics from parent statistics. |
static long |
getDataSizeFromColumnStats(long numRows,
List<ColStatistics> colStats)
Compute raw data size from column statistics |
static List<Long> |
getFileSizeForPartitions(HiveConf conf,
List<Partition> parts)
Find the bytes on disks occupied by list of partitions |
static long |
getFileSizeForTable(HiveConf conf,
Table table)
Find the bytes on disk occupied by a table |
static List<String> |
getFullQualifedColNameFromExprs(List<ExprNodeDesc> keyExprs,
Map<String,ExprNodeDesc> map)
Try to get fully qualified column name from expression node |
static String |
getFullyQualifiedColumnName(String tabName,
String colName)
Returns fully qualified name of column |
static String |
getFullyQualifiedColumnName(String dbName,
String tabName,
String colName)
Returns fully qualified name of column |
static String |
getFullyQualifiedColumnName(String dbName,
String tabName,
String partName,
String colName)
Returns fully qualified name of column |
static long |
getNumRows(Table table)
Get number of rows of a give table |
static Map<String,List<ColStatistics>> |
getPartColumnStats(Table table,
List<ColumnInfo> schema,
List<String> partNames,
List<String> neededColumns)
Get table level column statistics from metastore for needed columns |
static long |
getRawDataSize(Table table)
Get raw data size of a give table |
static long |
getSizeOfComplexTypes(HiveConf conf,
ObjectInspector oi)
Get the size of complex data types |
static long |
getSizeOfMap(StandardConstantMapObjectInspector scmoi)
Estimate the size of map object |
static long |
getSizeOfPrimitiveTypeArraysFromType(String colType,
int length)
Get the size of arrays of primitive types |
static long |
getSumIgnoreNegatives(List<Long> vals)
Get sum of all values in the list that are >0 |
static List<ColStatistics> |
getTableColumnStats(Table table,
List<ColumnInfo> schema,
List<String> neededColumns)
Get table level column statistics from metastore for needed columns |
static long |
getTotalSize(Table table)
Get total size of a give table |
static long |
getWritableSize(ObjectInspector oi,
Object value)
Get size of primitive data types based on their respective writable object inspector |
static String |
stripPrefixFromColumnName(String colName)
Remove KEY/VALUE prefix from column name |
Methods inherited from class java.lang.Object |
---|
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Constructor Detail |
---|
public StatsUtils()
Method Detail |
---|
public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, TableScanOperator tableScanOperator)
conf
- - hive configurationpartList
- - partition listtable
- - tabletableScanOperator
- - table scan operator
HiveException
public static int estimateRowSizeFromSchema(HiveConf conf, List<ColumnInfo> schema, List<String> neededColumns)
public static long getFileSizeForTable(HiveConf conf, Table table)
conf
- - hive conftable
- - table
public static List<Long> getFileSizeForPartitions(HiveConf conf, List<Partition> parts)
conf
- - hive confparts
- - partition list
public static long getSumIgnoreNegatives(List<Long> vals)
vals
- - list of values
public static ColStatistics getColStatistics(org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj cso, String tabName, String colName)
cso
- - ColumnStatisticsObjtabName
- - table namecolName
- - column name
public static List<ColStatistics> getTableColumnStats(Table table, List<ColumnInfo> schema, List<String> neededColumns)
table
- - tableschema
- - output schemaneededColumns
- - list of needed columns
public static Map<String,List<ColStatistics>> getPartColumnStats(Table table, List<ColumnInfo> schema, List<String> partNames, List<String> neededColumns)
table
- - tableschema
- - output schemaneededColumns
- - list of needed columns
public static long getAvgColLenOfVariableLengthTypes(HiveConf conf, ObjectInspector oi, String colType)
conf
- - hive confoi
- - object inspectorcolType
- - column type
public static long getSizeOfComplexTypes(HiveConf conf, ObjectInspector oi)
conf
- - hive confoi
- - object inspector
public static long getAvgColLenOfFixedLengthTypes(String colType)
colType
- - column type
public static long getSizeOfPrimitiveTypeArraysFromType(String colType, int length)
colType
- - column typelength
- - array length
public static long getSizeOfMap(StandardConstantMapObjectInspector scmoi)
scmoi
- - object inspector
public static long getWritableSize(ObjectInspector oi, Object value)
oi
- - object inspectorvalue
- - value
public static List<ColStatistics> getColStatisticsFromExprMap(HiveConf conf, Statistics parentStats, Map<String,ExprNodeDesc> colExprMap, RowSchema rowSchema)
conf
- - hive confparentStats
- - parent statisticscolExprMap
- - column expression maprowSchema
- - row schema
public static ColStatistics getColStatisticsFromExpression(HiveConf conf, Statistics parentStats, ExprNodeDesc end)
conf
- - hive confparentStats
- - parent statisticsend
- - expression nodes
public static long getNumRows(Table table)
public static long getRawDataSize(Table table)
public static long getTotalSize(Table table)
public static long getBasicStatForTable(Table table, String statType)
dbName
- - database nametabName
- - table namestatType
- - type of stats
public static List<Long> getBasicStatForPartitions(Table table, List<Partition> parts, String statType)
table
- - tableparts
- - partitionsstatType
- - type of stats
public static long getDataSizeFromColumnStats(long numRows, List<ColStatistics> colStats)
numRows
- - number of rowscolStats
- - column statistics
public static String stripPrefixFromColumnName(String colName)
colName
- - column name
public static String getFullyQualifiedColumnName(String tabName, String colName)
tabName
- colName
-
public static String getFullyQualifiedColumnName(String dbName, String tabName, String colName)
dbName
- tabName
- colName
-
public static String getFullyQualifiedColumnName(String dbName, String tabName, String partName, String colName)
dbName
- tabName
- partName
- colName
-
public static List<String> getFullQualifedColNameFromExprs(List<ExprNodeDesc> keyExprs, Map<String,ExprNodeDesc> map)
keyExprs
- - expression nodesmap
- - column expression map
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |