cascading.platform.hadoop
Class BaseHadoopPlatform

java.lang.Object
  extended by cascading.platform.TestPlatform
      extended by cascading.platform.hadoop.BaseHadoopPlatform
Direct Known Subclasses:
HadoopPlatform

public abstract class BaseHadoopPlatform
extends cascading.platform.TestPlatform


Field Summary
static FileSystem fileSys
           
static Configuration jobConf
           
protected  String logger
           
 int numMapTasks
           
 int numReduceTasks
           
static Map<Object,Object> properties
           
 
Fields inherited from class cascading.platform.TestPlatform
CLUSTER_TESTING_PROPERTY
 
Constructor Summary
BaseHadoopPlatform()
           
 
Method Summary
 void copyFromLocal(String inputFile)
           
 void copyToLocal(String outputFile)
           
 cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, cascading.tap.SinkMode mode)
           
 cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, cascading.tap.SinkMode mode)
           
 cascading.tap.Tap getDelimitedFile(String delimiter, String quote, cascading.scheme.util.FieldTypeResolver fieldTypeResolver, String filename, cascading.tap.SinkMode mode)
           
 cascading.flow.FlowProcess getFlowProcess()
           
 String getHiddenTemporaryPath()
           
 JobConf getJobConf()
           
 Comparator getLongComparator(boolean reverseSort)
           
 cascading.tap.Tap getPartitionTap(cascading.tap.Tap sink, cascading.tap.partition.Partition partition, int openThreshold)
           
 Map<Object,Object> getProperties()
           
 Comparator getStringComparator(boolean reverseSort)
           
 cascading.tap.Tap getTap(cascading.scheme.Scheme scheme, String filename, cascading.tap.SinkMode mode)
           
 cascading.tap.Tap getTemplateTap(cascading.tap.Tap sink, String pathTemplate, cascading.tuple.Fields fields, int openThreshold)
           
 cascading.tap.Tap getTemplateTap(cascading.tap.Tap sink, String pathTemplate, int openThreshold)
           
 cascading.scheme.Scheme getTestConfigDefScheme()
           
 cascading.scheme.Scheme getTestFailScheme()
           
 cascading.tap.Tap getTextFile(cascading.tuple.Fields sourceFields, cascading.tuple.Fields sinkFields, String filename, cascading.tap.SinkMode mode)
           
 boolean isHDFSAvailable()
           
 boolean isMapReduce()
           
 boolean remoteExists(String outputFile)
           
 boolean remoteRemove(String outputFile, boolean recursive)
           
protected  String safeFileName(String filename)
          Replaces characters, that are not allowed by HDFS with an "_".
 void setNumMapTasks(int numMapTasks)
           
 void setNumReduceTasks(int numReduceTasks)
           
 void tearDown()
           
 
Methods inherited from class cascading.platform.TestPlatform
getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getFlowConnector, getGlobalProperties, getName, getNumMappers, getNumMapTasks, getNumReducers, getNumReduceTasks, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isUseCluster, setNumMappers, setNumMapTasks, setNumReducers, setNumReduceTasks, setUp, setUseCluster
 
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

fileSys

public static transient FileSystem fileSys

jobConf

public static transient Configuration jobConf

properties

public static transient Map<Object,Object> properties

numMapTasks

public int numMapTasks

numReduceTasks

public int numReduceTasks

logger

protected String logger
Constructor Detail

BaseHadoopPlatform

public BaseHadoopPlatform()
Method Detail

isMapReduce

public boolean isMapReduce()
Overrides:
isMapReduce in class cascading.platform.TestPlatform

setNumMapTasks

public void setNumMapTasks(int numMapTasks)

setNumReduceTasks

public void setNumReduceTasks(int numReduceTasks)

getProperties

public Map<Object,Object> getProperties()
Specified by:
getProperties in class cascading.platform.TestPlatform

tearDown

public void tearDown()
Specified by:
tearDown in class cascading.platform.TestPlatform

getJobConf

public JobConf getJobConf()

isHDFSAvailable

public boolean isHDFSAvailable()

getFlowProcess

public cascading.flow.FlowProcess getFlowProcess()
Specified by:
getFlowProcess in class cascading.platform.TestPlatform

copyFromLocal

public void copyFromLocal(String inputFile)
                   throws IOException
Specified by:
copyFromLocal in class cascading.platform.TestPlatform
Throws:
IOException

copyToLocal

public void copyToLocal(String outputFile)
                 throws IOException
Specified by:
copyToLocal in class cascading.platform.TestPlatform
Throws:
IOException

remoteExists

public boolean remoteExists(String outputFile)
                     throws IOException
Specified by:
remoteExists in class cascading.platform.TestPlatform
Throws:
IOException

remoteRemove

public boolean remoteRemove(String outputFile,
                            boolean recursive)
                     throws IOException
Specified by:
remoteRemove in class cascading.platform.TestPlatform
Throws:
IOException

getTap

public cascading.tap.Tap getTap(cascading.scheme.Scheme scheme,
                                String filename,
                                cascading.tap.SinkMode mode)
Specified by:
getTap in class cascading.platform.TestPlatform

getTextFile

public cascading.tap.Tap getTextFile(cascading.tuple.Fields sourceFields,
                                     cascading.tuple.Fields sinkFields,
                                     String filename,
                                     cascading.tap.SinkMode mode)
Specified by:
getTextFile in class cascading.platform.TestPlatform

getDelimitedFile

public cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields,
                                          boolean hasHeader,
                                          String delimiter,
                                          String quote,
                                          Class[] types,
                                          String filename,
                                          cascading.tap.SinkMode mode)
Specified by:
getDelimitedFile in class cascading.platform.TestPlatform

getDelimitedFile

public cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields,
                                          boolean skipHeader,
                                          boolean writeHeader,
                                          String delimiter,
                                          String quote,
                                          Class[] types,
                                          String filename,
                                          cascading.tap.SinkMode mode)
Specified by:
getDelimitedFile in class cascading.platform.TestPlatform

getDelimitedFile

public cascading.tap.Tap getDelimitedFile(String delimiter,
                                          String quote,
                                          cascading.scheme.util.FieldTypeResolver fieldTypeResolver,
                                          String filename,
                                          cascading.tap.SinkMode mode)
Specified by:
getDelimitedFile in class cascading.platform.TestPlatform

getTemplateTap

public cascading.tap.Tap getTemplateTap(cascading.tap.Tap sink,
                                        String pathTemplate,
                                        int openThreshold)
Specified by:
getTemplateTap in class cascading.platform.TestPlatform

getTemplateTap

public cascading.tap.Tap getTemplateTap(cascading.tap.Tap sink,
                                        String pathTemplate,
                                        cascading.tuple.Fields fields,
                                        int openThreshold)
Specified by:
getTemplateTap in class cascading.platform.TestPlatform

getPartitionTap

public cascading.tap.Tap getPartitionTap(cascading.tap.Tap sink,
                                         cascading.tap.partition.Partition partition,
                                         int openThreshold)
Specified by:
getPartitionTap in class cascading.platform.TestPlatform

getTestConfigDefScheme

public cascading.scheme.Scheme getTestConfigDefScheme()
Specified by:
getTestConfigDefScheme in class cascading.platform.TestPlatform

getTestFailScheme

public cascading.scheme.Scheme getTestFailScheme()
Specified by:
getTestFailScheme in class cascading.platform.TestPlatform

getLongComparator

public Comparator getLongComparator(boolean reverseSort)
Specified by:
getLongComparator in class cascading.platform.TestPlatform

getStringComparator

public Comparator getStringComparator(boolean reverseSort)
Specified by:
getStringComparator in class cascading.platform.TestPlatform

getHiddenTemporaryPath

public String getHiddenTemporaryPath()
Specified by:
getHiddenTemporaryPath in class cascading.platform.TestPlatform

safeFileName

protected String safeFileName(String filename)
Replaces characters, that are not allowed by HDFS with an "_".

Parameters:
filename - The filename to make safe
Returns:
The filename with all non-supported characters removed.


Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.