cascading.flow.hadoop.util
Class HadoopUtil
java.lang.Object
cascading.flow.hadoop.util.HadoopUtil
public class HadoopUtil
- extends Object
Method Summary |
static Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> |
addToClassPath(org.apache.hadoop.mapred.JobConf config,
List<String> classpath)
Add to class path. |
static org.apache.hadoop.mapred.JobConf |
createJobConf(Map<Object,Object> properties,
org.apache.hadoop.mapred.JobConf defaultJobconf)
|
static Map<Object,Object> |
createProperties(org.apache.hadoop.conf.Configuration jobConf)
|
static byte[] |
decodeBytes(String string)
|
static
|
deserializeBase64(String string,
org.apache.hadoop.conf.Configuration conf,
Class<T> type)
This method deserializes the Base64 encoded String into an Object instance. |
static
|
deserializeBase64(String string,
org.apache.hadoop.conf.Configuration conf,
Class<T> type,
boolean decompress)
|
static String |
encodeBytes(byte[] bytes)
|
static Class |
findMainClass(Class defaultType)
|
static Map<String,String> |
getConfig(org.apache.hadoop.mapred.JobConf defaultConf,
org.apache.hadoop.mapred.JobConf updatedConf)
|
static Thread |
getHDFSShutdownHook()
|
static org.apache.hadoop.mapred.JobConf[] |
getJobConfs(org.apache.hadoop.mapred.JobConf job,
List<Map<String,String>> configs)
|
static cascading.flow.planner.PlatformInfo |
getPlatformInfo()
|
static void |
initLog4j(org.apache.hadoop.mapred.JobConf jobConf)
|
static
|
instantiateSerializer(org.apache.hadoop.conf.Configuration conf,
Class<T> type)
|
static org.apache.hadoop.mapred.JobConf |
mergeConf(org.apache.hadoop.mapred.JobConf job,
Map<String,String> config,
boolean directly)
|
static String |
readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf,
String id)
|
static org.apache.hadoop.mapred.JobConf |
removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf,
String... keys)
|
static boolean |
removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf,
String path)
|
static
|
serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf)
|
static
|
serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf,
boolean compress)
|
static void |
syncPaths(org.apache.hadoop.mapred.JobConf config,
Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths)
|
static String |
writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf,
String id,
String stepState)
|
Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
HadoopUtil
public HadoopUtil()
initLog4j
public static void initLog4j(org.apache.hadoop.mapred.JobConf jobConf)
createJobConf
public static org.apache.hadoop.mapred.JobConf createJobConf(Map<Object,Object> properties,
org.apache.hadoop.mapred.JobConf defaultJobconf)
createProperties
public static Map<Object,Object> createProperties(org.apache.hadoop.conf.Configuration jobConf)
getHDFSShutdownHook
public static Thread getHDFSShutdownHook()
encodeBytes
public static String encodeBytes(byte[] bytes)
decodeBytes
public static byte[] decodeBytes(String string)
instantiateSerializer
public static <T> ObjectSerializer instantiateSerializer(org.apache.hadoop.conf.Configuration conf,
Class<T> type)
throws ClassNotFoundException
- Throws:
ClassNotFoundException
serializeBase64
public static <T> String serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf)
throws IOException
- Throws:
IOException
serializeBase64
public static <T> String serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf,
boolean compress)
throws IOException
- Throws:
IOException
deserializeBase64
public static <T> T deserializeBase64(String string,
org.apache.hadoop.conf.Configuration conf,
Class<T> type)
throws IOException
- This method deserializes the Base64 encoded String into an Object instance.
- Parameters:
string
-
- Returns:
- an Object
- Throws:
IOException
deserializeBase64
public static <T> T deserializeBase64(String string,
org.apache.hadoop.conf.Configuration conf,
Class<T> type,
boolean decompress)
throws IOException
- Throws:
IOException
findMainClass
public static Class findMainClass(Class defaultType)
getConfig
public static Map<String,String> getConfig(org.apache.hadoop.mapred.JobConf defaultConf,
org.apache.hadoop.mapred.JobConf updatedConf)
getJobConfs
public static org.apache.hadoop.mapred.JobConf[] getJobConfs(org.apache.hadoop.mapred.JobConf job,
List<Map<String,String>> configs)
mergeConf
public static org.apache.hadoop.mapred.JobConf mergeConf(org.apache.hadoop.mapred.JobConf job,
Map<String,String> config,
boolean directly)
removePropertiesFrom
public static org.apache.hadoop.mapred.JobConf removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf,
String... keys)
removeStateFromDistCache
public static boolean removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf,
String path)
throws IOException
- Throws:
IOException
writeStateToDistCache
public static String writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf,
String id,
String stepState)
readStateFromDistCache
public static String readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf,
String id)
throws IOException
- Throws:
IOException
getPlatformInfo
public static cascading.flow.planner.PlatformInfo getPlatformInfo()
addToClassPath
public static Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> addToClassPath(org.apache.hadoop.mapred.JobConf config,
List<String> classpath)
- Add to class path.
- Parameters:
config
- the configclasspath
- the classpath
syncPaths
public static void syncPaths(org.apache.hadoop.mapred.JobConf config,
Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths)
Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.