public class HadoopUtil extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
static java.lang.String |
CASCADING_FLOW_EXECUTING |
Constructor and Description |
---|
HadoopUtil() |
Modifier and Type | Method and Description |
---|---|
static void |
addComparators(Configuration conf,
java.lang.String property,
java.util.Map<java.lang.String,cascading.tuple.Fields> map,
cascading.flow.planner.BaseFlowStep flowStep,
cascading.pipe.Group group) |
static void |
addComparators(Configuration conf,
java.lang.String property,
java.util.Map<java.lang.String,cascading.tuple.Fields> map,
cascading.tuple.Fields resolvedFields) |
static void |
addFields(Configuration conf,
java.lang.String property,
java.util.Map<java.lang.Integer,cascading.tuple.Fields> fields) |
static void |
addInputPath(Configuration conf,
Path path) |
static void |
addInputPaths(Configuration conf,
java.lang.Iterable<Path> paths) |
static JobConf |
asJobConfInstance(Configuration configuration) |
protected static <C extends Configuration> |
callCopyConstructor(java.lang.Class type,
Configuration parent) |
static <C> C |
copyConfiguration(C parent) |
static <C extends Configuration> |
copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties,
C dstConfiguration) |
static <C> C |
copyJobConf(C parentJobConf) |
static JobConf |
copyJobConf(JobConf parentJobConf) |
static JobConf |
createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf defaultJobconf) |
static cascading.flow.planner.PlatformInfo |
createPlatformInfo(java.lang.Class type,
java.lang.String attributePath,
java.lang.String platformName) |
static java.util.Map<java.lang.Object,java.lang.Object> |
createProperties(Configuration jobConf) |
static byte[] |
decodeBytes(java.lang.String string) |
static <T> T |
deserializeBase64(java.lang.String string,
Configuration conf,
java.lang.Class<T> type)
This method deserializes the Base64 encoded String into an Object instance.
|
static <T> T |
deserializeBase64(java.lang.String string,
Configuration conf,
java.lang.Class<T> type,
boolean decompress) |
static java.lang.String |
encodeBytes(byte[] bytes) |
static java.lang.Class |
findMainClass(java.lang.Class defaultType) |
static java.util.Map<Path,Path> |
getCommonPaths(java.util.Map<java.lang.String,Path> localPaths,
java.util.Map<java.lang.String,Path> remotePaths) |
static java.util.Map<java.lang.String,java.lang.String> |
getConfig(Configuration defaultConf,
Configuration updatedConf) |
static FileSystem |
getDefaultFS(Configuration config) |
static java.util.Map<java.lang.Integer,cascading.tuple.Fields> |
getFields(Configuration conf,
java.lang.String property) |
static java.lang.Thread |
getHDFSShutdownHook() |
static JobConf[] |
getJobConfs(Configuration job,
java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs) |
static LocalFileSystem |
getLocalFS(Configuration config) |
static Path |
getOutputPath(Configuration conf) |
static cascading.flow.planner.PlatformInfo |
getPlatformInfo() |
static cascading.flow.planner.PlatformInfo |
getPlatformInfo(java.lang.Class type,
java.lang.String attributePath,
java.lang.String platformName) |
static void |
initLog4j(Configuration configuration) |
static void |
initLog4j(JobConf configuration) |
static <T> ObjectSerializer |
instantiateSerializer(Configuration conf,
java.lang.Class<T> type) |
static boolean |
isInflow(Configuration conf) |
static boolean |
isLocal(Configuration conf) |
static boolean |
isYARN(Configuration conf) |
static <J extends Configuration> |
mergeConf(J job,
java.util.Map<java.lang.String,java.lang.String> config,
boolean directly) |
static java.lang.String |
pack(java.lang.Object object,
Configuration conf) |
static Configuration |
removePropertiesFrom(Configuration jobConf,
java.lang.String... keys) |
static boolean |
removeStateFromDistCache(Configuration conf,
java.lang.String path) |
static void |
resolvePaths(Configuration config,
java.util.Collection<java.lang.String> classpath,
java.lang.String remoteRoot,
java.lang.String resourceSubPath,
java.util.Map<java.lang.String,Path> localPaths,
java.util.Map<java.lang.String,Path> remotePaths) |
static <T> java.lang.String |
serializeBase64(T object,
Configuration conf) |
static <T> java.lang.String |
serializeBase64(T object,
Configuration conf,
boolean compress) |
static void |
setIsInflow(Configuration conf) |
static void |
setLocal(Configuration conf) |
static boolean |
setNewApi(Configuration conf,
java.lang.String className) |
static void |
setOutputPath(Configuration conf,
Path path) |
static java.util.Map<java.lang.String,java.lang.Long> |
syncPaths(Configuration config,
java.util.Map<Path,Path> commonPaths,
boolean syncTimes)
Copies paths from one local path to a remote path.
|
public static final java.lang.String CASCADING_FLOW_EXECUTING
public HadoopUtil()
public static void setIsInflow(Configuration conf)
public static boolean isInflow(Configuration conf)
public static void initLog4j(Configuration configuration)
public static JobConf asJobConfInstance(Configuration configuration)
public static <C> C copyJobConf(C parentJobConf)
public static JobConf copyJobConf(JobConf parentJobConf)
public static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf defaultJobconf)
public static <C> C copyConfiguration(C parent)
protected static <C extends Configuration> C callCopyConstructor(java.lang.Class type, Configuration parent)
public static <C extends Configuration> C copyConfiguration(java.util.Map<java.lang.Object,java.lang.Object> srcProperties, C dstConfiguration)
public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(Configuration jobConf)
public static java.lang.Thread getHDFSShutdownHook()
public static java.lang.String encodeBytes(byte[] bytes)
public static byte[] decodeBytes(java.lang.String string)
public static <T> ObjectSerializer instantiateSerializer(Configuration conf, java.lang.Class<T> type) throws java.lang.ClassNotFoundException
java.lang.ClassNotFoundException
public static <T> java.lang.String serializeBase64(T object, Configuration conf) throws java.io.IOException
java.io.IOException
public static <T> java.lang.String serializeBase64(T object, Configuration conf, boolean compress) throws java.io.IOException
java.io.IOException
public static <T> T deserializeBase64(java.lang.String string, Configuration conf, java.lang.Class<T> type) throws java.io.IOException
string
- java.io.IOException
public static <T> T deserializeBase64(java.lang.String string, Configuration conf, java.lang.Class<T> type, boolean decompress) throws java.io.IOException
java.io.IOException
public static java.lang.Class findMainClass(java.lang.Class defaultType)
public static java.util.Map<java.lang.String,java.lang.String> getConfig(Configuration defaultConf, Configuration updatedConf)
public static JobConf[] getJobConfs(Configuration job, java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)
public static <J extends Configuration> J mergeConf(J job, java.util.Map<java.lang.String,java.lang.String> config, boolean directly)
public static Configuration removePropertiesFrom(Configuration jobConf, java.lang.String... keys)
public static boolean removeStateFromDistCache(Configuration conf, java.lang.String path) throws java.io.IOException
java.io.IOException
public static cascading.flow.planner.PlatformInfo getPlatformInfo()
public static cascading.flow.planner.PlatformInfo getPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)
public static cascading.flow.planner.PlatformInfo createPlatformInfo(java.lang.Class type, java.lang.String attributePath, java.lang.String platformName)
public static java.util.Map<java.lang.String,java.lang.Long> syncPaths(Configuration config, java.util.Map<Path,Path> commonPaths, boolean syncTimes)
config
- commonPaths
- syncTimes
- public static java.util.Map<Path,Path> getCommonPaths(java.util.Map<java.lang.String,Path> localPaths, java.util.Map<java.lang.String,Path> remotePaths)
public static void resolvePaths(Configuration config, java.util.Collection<java.lang.String> classpath, java.lang.String remoteRoot, java.lang.String resourceSubPath, java.util.Map<java.lang.String,Path> localPaths, java.util.Map<java.lang.String,Path> remotePaths)
public static LocalFileSystem getLocalFS(Configuration config)
public static FileSystem getDefaultFS(Configuration config)
public static boolean isLocal(Configuration conf)
public static boolean isYARN(Configuration conf)
public static void setLocal(Configuration conf)
public static boolean setNewApi(Configuration conf, java.lang.String className)
public static void addInputPaths(Configuration conf, java.lang.Iterable<Path> paths)
public static void addInputPath(Configuration conf, Path path)
public static void setOutputPath(Configuration conf, Path path)
public static Path getOutputPath(Configuration conf)
public static java.lang.String pack(java.lang.Object object, Configuration conf)
public static void addFields(Configuration conf, java.lang.String property, java.util.Map<java.lang.Integer,cascading.tuple.Fields> fields)
public static java.util.Map<java.lang.Integer,cascading.tuple.Fields> getFields(Configuration conf, java.lang.String property) throws java.io.IOException
java.io.IOException
public static void addComparators(Configuration conf, java.lang.String property, java.util.Map<java.lang.String,cascading.tuple.Fields> map, cascading.flow.planner.BaseFlowStep flowStep, cascading.pipe.Group group)
public static void addComparators(Configuration conf, java.lang.String property, java.util.Map<java.lang.String,cascading.tuple.Fields> map, cascading.tuple.Fields resolvedFields)
Copyright © 2007-2015 Xplenty, Inc. All Rights Reserved.