public final class HiveFileFormatUtils
extends java.lang.Object
Modifier and Type | Method and Description |
---|---|
static boolean |
checkInputFormat(FileSystem fs,
HiveConf conf,
java.lang.Class<? extends InputFormat> inputFormatCls,
java.util.ArrayList<FileStatus> files)
checks if files are in same format as the given input format.
|
static java.util.List<java.lang.String> |
doGetAliasesFromPath(java.util.Map<java.lang.String,java.util.ArrayList<java.lang.String>> pathToAliases,
Path dir)
Get the list of aliases from the opeerator tree that are needed for the path
|
static java.util.List<Operator<? extends OperatorDesc>> |
doGetWorksFromPath(java.util.Map<java.lang.String,java.util.ArrayList<java.lang.String>> pathToAliases,
java.util.Map<java.lang.String,Operator<? extends OperatorDesc>> aliasToWork,
Path dir)
Get the list of operators from the operator tree that are needed for the path
|
static FileSinkOperator.RecordWriter |
getHiveRecordWriter(JobConf jc,
TableDesc tableInfo,
java.lang.Class<? extends Writable> outputClass,
FileSinkDesc conf,
Path outPath,
Reporter reporter) |
static java.lang.Class<? extends InputFormatChecker> |
getInputFormatChecker(java.lang.Class<?> inputFormat)
get an InputFormatChecker for a file format.
|
static Path |
getOutputFormatFinalPath(Path parent,
java.lang.String taskId,
JobConf jc,
HiveOutputFormat<?,?> hiveOutputFormat,
boolean isCompressed,
Path defaultFinalPath)
Deprecated.
|
static java.lang.Class<? extends HiveOutputFormat> |
getOutputFormatSubstitute(java.lang.Class<?> origin,
boolean storagehandlerflag)
get a OutputFormat's substitute HiveOutputFormat.
|
static PartitionDesc |
getPartitionDescFromPathRecursively(java.util.Map<java.lang.String,PartitionDesc> pathToPartitionInfo,
Path dir,
java.util.Map<java.util.Map<java.lang.String,PartitionDesc>,java.util.Map<java.lang.String,PartitionDesc>> cacheMap) |
static PartitionDesc |
getPartitionDescFromPathRecursively(java.util.Map<java.lang.String,PartitionDesc> pathToPartitionInfo,
Path dir,
java.util.Map<java.util.Map<java.lang.String,PartitionDesc>,java.util.Map<java.lang.String,PartitionDesc>> cacheMap,
boolean ignoreSchema) |
static java.lang.String |
getRealOutputFormatClassName()
get a RealOutputFormatClassName corresponding to the HivePassThroughOutputFormat
|
static FileSinkOperator.RecordWriter |
getRecordWriter(JobConf jc,
HiveOutputFormat<?,?> hiveOutputFormat,
java.lang.Class<? extends Writable> valueClass,
boolean isCompressed,
java.util.Properties tableProp,
Path outPath,
Reporter reporter) |
static void |
registerInputFormatChecker(java.lang.Class<? extends InputFormat> format,
java.lang.Class<? extends InputFormatChecker> checker)
register an InputFormatChecker for a given InputFormat.
|
static void |
registerOutputFormatSubstitute(java.lang.Class<? extends OutputFormat> origin,
java.lang.Class<? extends HiveOutputFormat> substitute)
register a substitute.
|
static void |
setRealOutputFormatClassName(java.lang.String destination)
set a RealOutputFormatClassName corresponding to the HivePassThroughOutputFormat
|
public static void registerOutputFormatSubstitute(java.lang.Class<? extends OutputFormat> origin, java.lang.Class<? extends HiveOutputFormat> substitute)
origin
- the class that need to be substitutedsubstitute
- public static java.lang.Class<? extends HiveOutputFormat> getOutputFormatSubstitute(java.lang.Class<?> origin, boolean storagehandlerflag)
public static java.lang.String getRealOutputFormatClassName()
public static void setRealOutputFormatClassName(java.lang.String destination)
@Deprecated public static Path getOutputFormatFinalPath(Path parent, java.lang.String taskId, JobConf jc, HiveOutputFormat<?,?> hiveOutputFormat, boolean isCompressed, Path defaultFinalPath) throws java.io.IOException
parent
- parent dir of the expected final output pathjc
- job configurationjava.io.IOException
public static void registerInputFormatChecker(java.lang.Class<? extends InputFormat> format, java.lang.Class<? extends InputFormatChecker> checker)
format
- the class that need to be substitutedchecker
- public static java.lang.Class<? extends InputFormatChecker> getInputFormatChecker(java.lang.Class<?> inputFormat)
public static boolean checkInputFormat(FileSystem fs, HiveConf conf, java.lang.Class<? extends InputFormat> inputFormatCls, java.util.ArrayList<FileStatus> files) throws HiveException
HiveException
public static FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jc, TableDesc tableInfo, java.lang.Class<? extends Writable> outputClass, FileSinkDesc conf, Path outPath, Reporter reporter) throws HiveException
HiveException
public static FileSinkOperator.RecordWriter getRecordWriter(JobConf jc, HiveOutputFormat<?,?> hiveOutputFormat, java.lang.Class<? extends Writable> valueClass, boolean isCompressed, java.util.Properties tableProp, Path outPath, Reporter reporter) throws java.io.IOException, HiveException
java.io.IOException
HiveException
public static PartitionDesc getPartitionDescFromPathRecursively(java.util.Map<java.lang.String,PartitionDesc> pathToPartitionInfo, Path dir, java.util.Map<java.util.Map<java.lang.String,PartitionDesc>,java.util.Map<java.lang.String,PartitionDesc>> cacheMap) throws java.io.IOException
java.io.IOException
public static PartitionDesc getPartitionDescFromPathRecursively(java.util.Map<java.lang.String,PartitionDesc> pathToPartitionInfo, Path dir, java.util.Map<java.util.Map<java.lang.String,PartitionDesc>,java.util.Map<java.lang.String,PartitionDesc>> cacheMap, boolean ignoreSchema) throws java.io.IOException
java.io.IOException
public static java.util.List<Operator<? extends OperatorDesc>> doGetWorksFromPath(java.util.Map<java.lang.String,java.util.ArrayList<java.lang.String>> pathToAliases, java.util.Map<java.lang.String,Operator<? extends OperatorDesc>> aliasToWork, Path dir)
pathToAliases
- mapping from path to aliasesaliasToWork
- The operator tree to be invoked for a given aliasdir
- The path to look forpublic static java.util.List<java.lang.String> doGetAliasesFromPath(java.util.Map<java.lang.String,java.util.ArrayList<java.lang.String>> pathToAliases, Path dir)
pathToAliases
- mapping from path to aliasesdir
- The path to look forCopyright © 2012 The Apache Software Foundation