public class HadoopFlow extends cascading.flow.BaseFlow<JobConf>
Flow
.
HadoopFlow must be created through a FlowConnector
sub-class instance.
If classpath paths are provided on the FlowDef
, the Hadoop distributed cache mechanism will be used
to augment the remote classpath.
Any path elements that are relative will be uploaded to HDFS, and the HDFS URI will be used on the JobConf. Note
all paths are added as "files" to the JobConf, not archives, so they aren't needlessly uncompressed cluster side.FlowConnector
Modifier | Constructor and Description |
---|---|
protected |
HadoopFlow() |
|
HadoopFlow(cascading.flow.planner.PlatformInfo platformInfo,
java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf jobConf,
cascading.flow.FlowDef flowDef) |
protected |
HadoopFlow(cascading.flow.planner.PlatformInfo platformInfo,
java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf jobConf,
java.lang.String name,
java.util.Map<java.lang.String,java.lang.String> flowDescriptor) |
Modifier and Type | Method and Description |
---|---|
protected void |
copyToDistributedCache() |
JobConf |
getConfig() |
java.util.Map<java.lang.Object,java.lang.Object> |
getConfigAsProperties() |
JobConf |
getConfigCopy() |
cascading.flow.FlowProcess<JobConf> |
getFlowProcess() |
protected int |
getMaxNumParallelSteps() |
java.lang.String |
getProperty(java.lang.String key)
Method getProperty returns the value associated with the given key from the underlying properties system.
|
protected long |
getTotalSliceCPUMilliSeconds() |
protected void |
initConfig(java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf parentConfig) |
protected void |
initFromProperties(java.util.Map<java.lang.Object,java.lang.Object> properties) |
protected void |
internalClean(boolean stop) |
protected void |
internalShutdown() |
protected void |
internalStart() |
boolean |
isPreserveTemporaryFiles()
Method isPreserveTemporaryFiles returns false if temporary files will be cleaned when this Flow completes.
|
protected JobConf |
newConfig(JobConf defaultConfig) |
protected void |
registerHadoopShutdownHook() |
protected void |
setConfigProperty(JobConf config,
java.lang.Object key,
java.lang.Object value) |
boolean |
stepsAreLocal() |
addListener, addPlannerProperties, addSessionProperties, addStepListener, areSinksStale, areSourcesNewer, cleanup, complete, createConfig, createFlowCanonicalHash, createFlowStats, createFlowThread, createPrepareFlowStats, deleteCheckpointsIfNotUpdate, deleteCheckpointsIfReplace, deleteSinks, deleteSinksIfNotUpdate, deleteSinksIfReplace, deleteTrapsIfNotUpdate, deleteTrapsIfReplace, fireOnCompleted, fireOnStarting, fireOnStopping, fireOnThrowable, fireOnThrowable, getCascadeID, getCascadingServices, getCheckpointNames, getCheckpoints, getCheckpointsCollection, getClassPath, getClientState, getEligibleJobsSize, getFieldsFor, getFlowCanonicalHash, getFlowDescriptor, getFlowElementGraph, getFlowSession, getFlowSkipStrategy, getFlowStats, getFlowStepGraph, getFlowSteps, getFlowStepStrategy, getHolder, getID, getJobMapCallables, getName, getPlannerInfo, getPlatformInfo, getRunID, getSink, getSink, getSinkModified, getSinkNames, getSinks, getSinksCollection, getSource, getSourceNames, getSources, getSourcesCollection, getSpawnStrategy, getStats, getSubmitPriority, getTags, getTrapNames, getTraps, getTrapsCollection, handleExecutorShutdown, hasListeners, hasStepListeners, initialize, initializeChildStats, initializeNewJobsMap, initSteps, internalStopAllJobs, isDebugEnabled, isInfoEnabled, isJobsMapInitialized, isSkipFlow, isStopJobsOnExit, logDebug, logError, logError, logInfo, logWarn, logWarn, logWarn, openSink, openSink, openSource, openSource, openTapForRead, openTapForWrite, openTrap, openTrap, prepare, presentSinkFields, presentSourceFields, registerShutdownHook, removeListener, removeListeners, removeStepListener, resourceExists, retrieveSinkFields, retrieveSourceFields, setCascade, setCheckpoints, setFlowElementGraph, setFlowSkipStrategy, setFlowStepGraph, setFlowStepStrategy, setName, setPlannerInfo, setSinks, setSources, setSpawnStrategy, setSubmitPriority, setTraps, spawnSteps, start, stop, toString, updateJobsMap, updateSchemes, writeDOT, writeStepsDOT
protected HadoopFlow()
protected HadoopFlow(cascading.flow.planner.PlatformInfo platformInfo, java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf jobConf, java.lang.String name, java.util.Map<java.lang.String,java.lang.String> flowDescriptor)
public HadoopFlow(cascading.flow.planner.PlatformInfo platformInfo, java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf jobConf, cascading.flow.FlowDef flowDef)
protected void initFromProperties(java.util.Map<java.lang.Object,java.lang.Object> properties)
initFromProperties
in class cascading.flow.BaseFlow<JobConf>
protected void initConfig(java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf parentConfig)
initConfig
in class cascading.flow.BaseFlow<JobConf>
protected void setConfigProperty(JobConf config, java.lang.Object key, java.lang.Object value)
setConfigProperty
in class cascading.flow.BaseFlow<JobConf>
protected JobConf newConfig(JobConf defaultConfig)
newConfig
in class cascading.flow.BaseFlow<JobConf>
public JobConf getConfigCopy()
public java.util.Map<java.lang.Object,java.lang.Object> getConfigAsProperties()
public java.lang.String getProperty(java.lang.String key)
key
- of type Stringpublic cascading.flow.FlowProcess<JobConf> getFlowProcess()
public boolean isPreserveTemporaryFiles()
protected void internalStart()
internalStart
in class cascading.flow.BaseFlow<JobConf>
protected void registerHadoopShutdownHook()
protected void copyToDistributedCache()
public boolean stepsAreLocal()
protected void internalClean(boolean stop)
internalClean
in class cascading.flow.BaseFlow<JobConf>
protected void internalShutdown()
internalShutdown
in class cascading.flow.BaseFlow<JobConf>
protected int getMaxNumParallelSteps()
getMaxNumParallelSteps
in class cascading.flow.BaseFlow<JobConf>
protected long getTotalSliceCPUMilliSeconds()
getTotalSliceCPUMilliSeconds
in class cascading.flow.BaseFlow<JobConf>
Copyright © 2007-2015 Xplenty, Inc. All Rights Reserved.