public abstract class BaseHadoopPlatform<Config extends Configuration> extends cascading.platform.TestPlatform
Modifier and Type | Field and Description |
---|---|
static Configuration |
configuration |
static FileSystem |
fileSys |
protected java.lang.String |
logger |
static java.util.Map<java.lang.Object,java.lang.Object> |
properties |
Constructor and Description |
---|
BaseHadoopPlatform() |
Modifier and Type | Method and Description |
---|---|
void |
copyFromLocal(java.lang.String inputFile) |
void |
copyToLocal(java.lang.String outputFile) |
abstract Config |
getConfiguration() |
cascading.tap.Tap |
getDelimitedFile(cascading.tuple.Fields fields,
boolean skipHeader,
boolean writeHeader,
java.lang.String delimiter,
java.lang.String quote,
java.lang.Class[] types,
java.lang.String filename,
cascading.tap.SinkMode mode) |
cascading.tap.Tap |
getDelimitedFile(cascading.tuple.Fields fields,
boolean hasHeader,
java.lang.String delimiter,
java.lang.String quote,
java.lang.Class[] types,
java.lang.String filename,
cascading.tap.SinkMode mode) |
cascading.tap.Tap |
getDelimitedFile(java.lang.String delimiter,
java.lang.String quote,
cascading.scheme.util.FieldTypeResolver fieldTypeResolver,
java.lang.String filename,
cascading.tap.SinkMode mode) |
java.lang.String |
getHiddenTemporaryPath() |
java.util.Comparator |
getLongComparator(boolean reverseSort) |
cascading.tap.Tap |
getPartitionTap(cascading.tap.Tap sink,
cascading.tap.partition.Partition partition,
int openThreshold) |
java.util.Map<java.lang.Object,java.lang.Object> |
getProperties() |
java.util.Comparator |
getStringComparator(boolean reverseSort) |
cascading.tap.Tap |
getTap(cascading.scheme.Scheme scheme,
java.lang.String filename,
cascading.tap.SinkMode mode) |
cascading.scheme.Scheme |
getTestConfigDefScheme() |
cascading.scheme.Scheme |
getTestFailScheme() |
cascading.tap.Tap |
getTextFile(cascading.tuple.Fields sourceFields,
cascading.tuple.Fields sinkFields,
java.lang.String filename,
cascading.tap.SinkMode mode) |
boolean |
isHDFSAvailable() |
boolean |
isMapReduce() |
boolean |
remoteExists(java.lang.String outputFile) |
boolean |
remoteRemove(java.lang.String outputFile,
boolean recursive) |
protected java.lang.String |
safeFileName(java.lang.String filename)
Replaces characters, that are not allowed by HDFS with an "_".
|
void |
setNumGatherPartitions(int numGatherPartitions) |
void |
setNumMappers(int numMapTasks) |
void |
setNumReducers(int numReduceTasks) |
void |
tearDown() |
getApplicationJar, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getDelimitedFile, getFlowConnector, getFlowConnector, getFlowProcess, getGlobalProperties, getName, getNumGatherPartitions, getNumGatherPartitionTasks, getNumMappers, getNumMapTasks, getNumReducers, getNumReduceTasks, getTabDelimitedFile, getTabDelimitedFile, getTextFile, getTextFile, getTextFile, getTextFile, isDAG, isUseCluster, setNumGatherPartitionTasks, setNumMapTasks, setNumReduceTasks, setUp, setUseCluster, supportsGroupByAfterMerge
public static transient FileSystem fileSys
public static transient Configuration configuration
public static transient java.util.Map<java.lang.Object,java.lang.Object> properties
protected java.lang.String logger
public BaseHadoopPlatform()
public boolean isMapReduce()
isMapReduce
in class cascading.platform.TestPlatform
public void setNumMappers(int numMapTasks)
setNumMappers
in class cascading.platform.TestPlatform
public void setNumReducers(int numReduceTasks)
setNumReducers
in class cascading.platform.TestPlatform
public void setNumGatherPartitions(int numGatherPartitions)
setNumGatherPartitions
in class cascading.platform.TestPlatform
public java.util.Map<java.lang.Object,java.lang.Object> getProperties()
getProperties
in class cascading.platform.TestPlatform
public void tearDown()
tearDown
in class cascading.platform.TestPlatform
public abstract Config getConfiguration()
public boolean isHDFSAvailable()
public void copyFromLocal(java.lang.String inputFile) throws java.io.IOException
copyFromLocal
in class cascading.platform.TestPlatform
java.io.IOException
public void copyToLocal(java.lang.String outputFile) throws java.io.IOException
copyToLocal
in class cascading.platform.TestPlatform
java.io.IOException
public boolean remoteExists(java.lang.String outputFile) throws java.io.IOException
remoteExists
in class cascading.platform.TestPlatform
java.io.IOException
public boolean remoteRemove(java.lang.String outputFile, boolean recursive) throws java.io.IOException
remoteRemove
in class cascading.platform.TestPlatform
java.io.IOException
public cascading.tap.Tap getTap(cascading.scheme.Scheme scheme, java.lang.String filename, cascading.tap.SinkMode mode)
getTap
in class cascading.platform.TestPlatform
public cascading.tap.Tap getTextFile(cascading.tuple.Fields sourceFields, cascading.tuple.Fields sinkFields, java.lang.String filename, cascading.tap.SinkMode mode)
getTextFile
in class cascading.platform.TestPlatform
public cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields, boolean hasHeader, java.lang.String delimiter, java.lang.String quote, java.lang.Class[] types, java.lang.String filename, cascading.tap.SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public cascading.tap.Tap getDelimitedFile(cascading.tuple.Fields fields, boolean skipHeader, boolean writeHeader, java.lang.String delimiter, java.lang.String quote, java.lang.Class[] types, java.lang.String filename, cascading.tap.SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public cascading.tap.Tap getDelimitedFile(java.lang.String delimiter, java.lang.String quote, cascading.scheme.util.FieldTypeResolver fieldTypeResolver, java.lang.String filename, cascading.tap.SinkMode mode)
getDelimitedFile
in class cascading.platform.TestPlatform
public cascading.tap.Tap getPartitionTap(cascading.tap.Tap sink, cascading.tap.partition.Partition partition, int openThreshold)
getPartitionTap
in class cascading.platform.TestPlatform
public cascading.scheme.Scheme getTestConfigDefScheme()
getTestConfigDefScheme
in class cascading.platform.TestPlatform
public cascading.scheme.Scheme getTestFailScheme()
getTestFailScheme
in class cascading.platform.TestPlatform
public java.util.Comparator getLongComparator(boolean reverseSort)
getLongComparator
in class cascading.platform.TestPlatform
public java.util.Comparator getStringComparator(boolean reverseSort)
getStringComparator
in class cascading.platform.TestPlatform
public java.lang.String getHiddenTemporaryPath()
getHiddenTemporaryPath
in class cascading.platform.TestPlatform
protected java.lang.String safeFileName(java.lang.String filename)
filename
- The filename to make safeCopyright © 2007-2015 Xplenty, Inc. All Rights Reserved.