public abstract class BaseHadoopPlatform<Config extends Configuration> extends cascading.platform.TestPlatform
Modifier and Type | Field and Description |
---|---|
static Configuration |
configuration |
static FileSystem |
fileSys |
protected String |
logger |
static Map<Object,Object> |
properties |
CLUSTER_TESTING_PROPERTY, numGatherPartitions, numMappers, numReducers, PLATFORM_PREFIX
Constructor and Description |
---|
BaseHadoopPlatform() |
Modifier and Type | Method and Description |
---|---|
void |
copyFromLocal(String inputFile) |
void |
copyToLocal(String outputFile) |
abstract Config |
getConfiguration() |
Tap |
getDelimitedFile(Fields fields,
boolean skipHeader,
boolean writeHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode) |
Tap |
getDelimitedFile(Fields fields,
boolean hasHeader,
String delimiter,
String quote,
Class[] types,
String filename,
SinkMode mode) |
Tap |
getDelimitedFile(String delimiter,
String quote,
FieldTypeResolver fieldTypeResolver,
String filename,
SinkMode mode) |
String |
getHiddenTemporaryPath() |
Comparator |
getLongComparator(boolean reverseSort) |
Tap |
getPartitionTap(Tap sink,
Partition partition,
int openThreshold) |
Map<Object,Object> |
getProperties() |
Comparator |
getStringComparator(boolean reverseSort) |
Tap |
getTap(Scheme scheme,
String filename,
SinkMode mode) |
Scheme |
getTestConfigDefScheme() |
Scheme |
getTestFailScheme() |
Tap |
getTextFile(Fields sourceFields,
Fields sinkFields,
String filename,
SinkMode mode) |
boolean |
isHDFSAvailable() |
boolean |
isMapReduce() |
boolean |
remoteExists(String outputFile) |
boolean |
remoteRemove(String outputFile,
boolean recursive) |
protected String |
safeFileName(String filename)
Replaces characters, that are not allowed by HDFS with an "_".
|
void |
setNumGatherPartitions(int numGatherPartitions) |
void |
setNumMappers(int numMapTasks) |
void |
setNumReducers(int numReduceTasks) |
void |
tearDown() |
getApplicationJar, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getFlowConnector, getFlowProcess, getGlobalProperties, getName, getNumGatherPartitions, getNumGatherPartitionTasks, getNumMappers, getNumMapTasks, getNumReducers, getNumReduceTasks, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isDAG, isUseCluster, setNumGatherPartitionTasks, setNumMapTasks, setNumReduceTasks, setUp, setUseCluster, supportsGroupByAfterMerge
public static transient FileSystem fileSys
public static transient Configuration configuration
public static transient Map<Object,Object> properties
public BaseHadoopPlatform()
public boolean isMapReduce()
isMapReduce
in class cascading.platform.TestPlatform
public void setNumMappers(int numMapTasks)
setNumMappers
in class cascading.platform.TestPlatform
public void setNumReducers(int numReduceTasks)
setNumReducers
in class cascading.platform.TestPlatform
public void setNumGatherPartitions(int numGatherPartitions)
setNumGatherPartitions
in class cascading.platform.TestPlatform
public Map<Object,Object> getProperties()
getProperties
in class cascading.platform.TestPlatform
public void tearDown()
tearDown
in class cascading.platform.TestPlatform
public abstract Config getConfiguration()
public boolean isHDFSAvailable()
public void copyFromLocal(String inputFile) throws IOException
copyFromLocal
in class cascading.platform.TestPlatform
IOException
public void copyToLocal(String outputFile) throws IOException
copyToLocal
in class cascading.platform.TestPlatform
IOException
public boolean remoteExists(String outputFile) throws IOException
remoteExists
in class cascading.platform.TestPlatform
IOException
public boolean remoteRemove(String outputFile, boolean recursive) throws IOException
remoteRemove
in class cascading.platform.TestPlatform
IOException
public Tap getTap(Scheme scheme, String filename, SinkMode mode)
getTap
in class cascading.platform.TestPlatform
public Tap getTextFile(Fields sourceFields, Fields sinkFields, String filename, SinkMode mode)
getTextFile
in class cascading.platform.TestPlatform
public Tap getDelimitedFile(Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public Tap getDelimitedFile(Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public Tap getDelimitedFile(String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public Tap getPartitionTap(Tap sink, Partition partition, int openThreshold)
getPartitionTap
in class cascading.platform.TestPlatform
public Scheme getTestConfigDefScheme()
getTestConfigDefScheme
in class cascading.platform.TestPlatform
public Scheme getTestFailScheme()
getTestFailScheme
in class cascading.platform.TestPlatform
public Comparator getLongComparator(boolean reverseSort)
getLongComparator
in class cascading.platform.TestPlatform
public Comparator getStringComparator(boolean reverseSort)
getStringComparator
in class cascading.platform.TestPlatform
public String getHiddenTemporaryPath()
getHiddenTemporaryPath
in class cascading.platform.TestPlatform
protected String safeFileName(String filename)
filename
- The filename to make safeCopyright © 2007-2015 Concurrent, Inc. All Rights Reserved.