cascading.flow.hadoop.util
Class HadoopUtil

java.lang.Object
  extended by cascading.flow.hadoop.util.HadoopUtil

public class HadoopUtil
extends Object


Constructor Summary
HadoopUtil()
           
 
Method Summary
static Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> addToClassPath(org.apache.hadoop.mapred.JobConf config, List<String> classpath)
          Add to class path.
static org.apache.hadoop.mapred.JobConf createJobConf(Map<Object,Object> properties, org.apache.hadoop.mapred.JobConf defaultJobconf)
           
static Map<Object,Object> createProperties(org.apache.hadoop.conf.Configuration jobConf)
           
static byte[] decodeBytes(String string)
           
static
<T> T
deserializeBase64(String string, org.apache.hadoop.conf.Configuration conf, Class<T> type)
          This method deserializes the Base64 encoded String into an Object instance.
static
<T> T
deserializeBase64(String string, org.apache.hadoop.conf.Configuration conf, Class<T> type, boolean decompress)
           
static String encodeBytes(byte[] bytes)
           
static Class findMainClass(Class defaultType)
           
static Map<String,String> getConfig(org.apache.hadoop.mapred.JobConf defaultConf, org.apache.hadoop.mapred.JobConf updatedConf)
           
static Thread getHDFSShutdownHook()
           
static org.apache.hadoop.mapred.JobConf[] getJobConfs(org.apache.hadoop.mapred.JobConf job, List<Map<String,String>> configs)
           
static cascading.flow.planner.PlatformInfo getPlatformInfo()
           
static void initLog4j(org.apache.hadoop.mapred.JobConf jobConf)
           
static
<T> ObjectSerializer
instantiateSerializer(org.apache.hadoop.conf.Configuration conf, Class<T> type)
           
static org.apache.hadoop.mapred.JobConf mergeConf(org.apache.hadoop.mapred.JobConf job, Map<String,String> config, boolean directly)
           
static String readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf, String id)
           
static org.apache.hadoop.mapred.JobConf removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf, String... keys)
           
static boolean removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf, String path)
           
static
<T> String
serializeBase64(T object, org.apache.hadoop.mapred.JobConf conf)
           
static
<T> String
serializeBase64(T object, org.apache.hadoop.mapred.JobConf conf, boolean compress)
           
static void syncPaths(org.apache.hadoop.mapred.JobConf config, Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths)
           
static String writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf, String id, String stepState)
           
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Constructor Detail

HadoopUtil

public HadoopUtil()
Method Detail

initLog4j

public static void initLog4j(org.apache.hadoop.mapred.JobConf jobConf)

createJobConf

public static org.apache.hadoop.mapred.JobConf createJobConf(Map<Object,Object> properties,
                                                             org.apache.hadoop.mapred.JobConf defaultJobconf)

createProperties

public static Map<Object,Object> createProperties(org.apache.hadoop.conf.Configuration jobConf)

getHDFSShutdownHook

public static Thread getHDFSShutdownHook()

encodeBytes

public static String encodeBytes(byte[] bytes)

decodeBytes

public static byte[] decodeBytes(String string)

instantiateSerializer

public static <T> ObjectSerializer instantiateSerializer(org.apache.hadoop.conf.Configuration conf,
                                                         Class<T> type)
                                              throws ClassNotFoundException
Throws:
ClassNotFoundException

serializeBase64

public static <T> String serializeBase64(T object,
                                         org.apache.hadoop.mapred.JobConf conf)
                              throws IOException
Throws:
IOException

serializeBase64

public static <T> String serializeBase64(T object,
                                         org.apache.hadoop.mapred.JobConf conf,
                                         boolean compress)
                              throws IOException
Throws:
IOException

deserializeBase64

public static <T> T deserializeBase64(String string,
                                      org.apache.hadoop.conf.Configuration conf,
                                      Class<T> type)
                           throws IOException
This method deserializes the Base64 encoded String into an Object instance.

Parameters:
string -
Returns:
an Object
Throws:
IOException

deserializeBase64

public static <T> T deserializeBase64(String string,
                                      org.apache.hadoop.conf.Configuration conf,
                                      Class<T> type,
                                      boolean decompress)
                           throws IOException
Throws:
IOException

findMainClass

public static Class findMainClass(Class defaultType)

getConfig

public static Map<String,String> getConfig(org.apache.hadoop.mapred.JobConf defaultConf,
                                           org.apache.hadoop.mapred.JobConf updatedConf)

getJobConfs

public static org.apache.hadoop.mapred.JobConf[] getJobConfs(org.apache.hadoop.mapred.JobConf job,
                                                             List<Map<String,String>> configs)

mergeConf

public static org.apache.hadoop.mapred.JobConf mergeConf(org.apache.hadoop.mapred.JobConf job,
                                                         Map<String,String> config,
                                                         boolean directly)

removePropertiesFrom

public static org.apache.hadoop.mapred.JobConf removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf,
                                                                    String... keys)

removeStateFromDistCache

public static boolean removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf,
                                               String path)
                                        throws IOException
Throws:
IOException

writeStateToDistCache

public static String writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf,
                                           String id,
                                           String stepState)

readStateFromDistCache

public static String readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf,
                                            String id)
                                     throws IOException
Throws:
IOException

getPlatformInfo

public static cascading.flow.planner.PlatformInfo getPlatformInfo()

addToClassPath

public static Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> addToClassPath(org.apache.hadoop.mapred.JobConf config,
                                                                                      List<String> classpath)
Add to class path.

Parameters:
config - the config
classpath - the classpath

syncPaths

public static void syncPaths(org.apache.hadoop.mapred.JobConf config,
                             Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths)


Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.