cascading.flow.hadoop.util
Class HadoopUtil
java.lang.Object
cascading.flow.hadoop.util.HadoopUtil
public class HadoopUtil
- extends Object
Method Summary |
static JobConf |
createJobConf(Map<Object,Object> properties,
JobConf defaultJobconf)
|
static Map<Object,Object> |
createProperties(Configuration jobConf)
|
static byte[] |
decodeBytes(String string)
|
static
|
deserializeBase64(String string,
Configuration conf,
Class<T> type)
This method deserializes the Base64 encoded String into an Object instance. |
static
|
deserializeBase64(String string,
Configuration conf,
Class<T> type,
boolean decompress)
|
static String |
encodeBytes(byte[] bytes)
|
static Class |
findMainClass(Class defaultType)
|
static Map<String,String> |
getConfig(JobConf defaultConf,
JobConf updatedConf)
|
static Thread |
getHDFSShutdownHook()
|
static JobConf[] |
getJobConfs(JobConf job,
List<Map<String,String>> configs)
|
static void |
initLog4j(JobConf jobConf)
|
static
|
instantiateSerializer(Configuration conf,
Class<T> type)
|
static JobConf |
mergeConf(JobConf job,
Map<String,String> config,
boolean directly)
|
static String |
readStateFromDistCache(JobConf jobConf,
String id)
|
static JobConf |
removePropertiesFrom(JobConf jobConf,
String... keys)
|
static boolean |
removeStateFromDistCache(JobConf conf,
String path)
|
static
|
serializeBase64(T object,
JobConf conf)
|
static
|
serializeBase64(T object,
JobConf conf,
boolean compress)
|
static String |
writeStateToDistCache(JobConf conf,
String id,
String stepState)
|
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
HadoopUtil
public HadoopUtil()
initLog4j
public static void initLog4j(JobConf jobConf)
createJobConf
public static JobConf createJobConf(Map<Object,Object> properties,
JobConf defaultJobconf)
createProperties
public static Map<Object,Object> createProperties(Configuration jobConf)
getHDFSShutdownHook
public static Thread getHDFSShutdownHook()
encodeBytes
public static String encodeBytes(byte[] bytes)
decodeBytes
public static byte[] decodeBytes(String string)
instantiateSerializer
public static <T> ObjectSerializer instantiateSerializer(Configuration conf,
Class<T> type)
throws ClassNotFoundException
- Throws:
ClassNotFoundException
serializeBase64
public static <T> String serializeBase64(T object,
JobConf conf)
throws IOException
- Throws:
IOException
serializeBase64
public static <T> String serializeBase64(T object,
JobConf conf,
boolean compress)
throws IOException
- Throws:
IOException
deserializeBase64
public static <T> T deserializeBase64(String string,
Configuration conf,
Class<T> type)
throws IOException
- This method deserializes the Base64 encoded String into an Object instance.
- Parameters:
string
-
- Returns:
- an Object
- Throws:
IOException
deserializeBase64
public static <T> T deserializeBase64(String string,
Configuration conf,
Class<T> type,
boolean decompress)
throws IOException
- Throws:
IOException
findMainClass
public static Class findMainClass(Class defaultType)
getConfig
public static Map<String,String> getConfig(JobConf defaultConf,
JobConf updatedConf)
getJobConfs
public static JobConf[] getJobConfs(JobConf job,
List<Map<String,String>> configs)
mergeConf
public static JobConf mergeConf(JobConf job,
Map<String,String> config,
boolean directly)
removePropertiesFrom
public static JobConf removePropertiesFrom(JobConf jobConf,
String... keys)
removeStateFromDistCache
public static boolean removeStateFromDistCache(JobConf conf,
String path)
throws IOException
- Throws:
IOException
writeStateToDistCache
public static String writeStateToDistCache(JobConf conf,
String id,
String stepState)
readStateFromDistCache
public static String readStateFromDistCache(JobConf jobConf,
String id)
throws IOException
- Throws:
IOException
Copyright © 2007-2012 Concurrent, Inc. All Rights Reserved.