cascading.flow.hadoop
Class HadoopFlowStep

java.lang.Object
  extended by cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf>
      extended by cascading.flow.hadoop.HadoopFlowStep
All Implemented Interfaces:
FlowStep<org.apache.hadoop.mapred.JobConf>, Serializable
Direct Known Subclasses:
MapReduceFlowStep

public class HadoopFlowStep
extends cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf>

See Also:
Serialized Form

Field Summary
 
Fields inherited from class cascading.flow.planner.BaseFlowStep
accumulatedSourcesByJoin, sinks, sources, streamedSourceByJoin, tempSink
 
Fields inherited from interface cascading.flow.FlowStep
CASCADING_FLOW_STEP_ID
 
Constructor Summary
HadoopFlowStep(String name, int stepNum)
           
 
Method Summary
 void clean(org.apache.hadoop.mapred.JobConf config)
          Method clean removes any temporary files used by this FlowStep instance.
protected  cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf> createFlowStepJob(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess, org.apache.hadoop.mapred.JobConf parentConfig)
           
 org.apache.hadoop.mapred.JobConf getInitializedConfig(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess, org.apache.hadoop.mapred.JobConf parentConfig)
           
 Tap getMapperTrap(String name)
           
 Map<String,Tap> getMapperTraps()
           
 Tap getReducerTrap(String name)
           
 Map<String,Tap> getReducerTraps()
           
 Tap getTapForID(Set<Tap> taps, String id)
           
 Tap getTrap(String name)
           
 Set<Tap> getTraps()
           
protected  void initFromSink(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess, org.apache.hadoop.mapred.JobConf conf)
           
protected  void initFromSources(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess, org.apache.hadoop.mapred.JobConf conf)
           
protected  void initFromTraps(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess, org.apache.hadoop.mapred.JobConf conf)
           
 boolean isHadoopLocalMode(org.apache.hadoop.mapred.JobConf conf)
           
 
Methods inherited from class cascading.flow.planner.BaseFlowStep
addAccumulatedSourceFor, addGroup, addSink, addSource, addStreamedSourceFor, clean, commitSinks, containsPipeNamed, createClientState, equals, getAllAccumulatedSources, getAllOperations, getConfig, getFlow, getFlowID, getFlowName, getFlowStepJob, getFlowStepStats, getGraph, getGroup, getGroups, getID, getJoinTributariesBetween, getName, getNextFlowElement, getNextScope, getNextScopes, getPreviousScopes, getScopeFor, getSink, getSinkName, getSinks, getSinkWith, getSourceName, getSources, getSourceWith, getStepDisplayName, getStepDisplayName, getStepNum, getStreamedSourceByJoin, getSubmitPriority, getSuccessors, getTopologicalOrderIterator, hashCode, initConfFromProcessConfigDef, isDebugEnabled, isInfoEnabled, logDebug, logError, logInfo, logWarn, logWarn, rollbackSinks, setConf, setFlow, setFlowName, setSubmitPriority, toString
 
Methods inherited from class java.lang.Object
clone, finalize, getClass, notify, notifyAll, wait, wait, wait
 

Constructor Detail

HadoopFlowStep

public HadoopFlowStep(String name,
                      int stepNum)
Method Detail

getInitializedConfig

public org.apache.hadoop.mapred.JobConf getInitializedConfig(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess,
                                                             org.apache.hadoop.mapred.JobConf parentConfig)
Specified by:
getInitializedConfig in class cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf>

isHadoopLocalMode

public boolean isHadoopLocalMode(org.apache.hadoop.mapred.JobConf conf)

createFlowStepJob

protected cascading.flow.planner.FlowStepJob<org.apache.hadoop.mapred.JobConf> createFlowStepJob(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess,
                                                                                                 org.apache.hadoop.mapred.JobConf parentConfig)
Specified by:
createFlowStepJob in class cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf>

clean

public void clean(org.apache.hadoop.mapred.JobConf config)
Method clean removes any temporary files used by this FlowStep instance. It will log any IOExceptions thrown.

Specified by:
clean in class cascading.flow.planner.BaseFlowStep<org.apache.hadoop.mapred.JobConf>
Parameters:
config - of type JobConf

initFromSources

protected void initFromSources(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess,
                               org.apache.hadoop.mapred.JobConf conf)

getTapForID

public Tap getTapForID(Set<Tap> taps,
                       String id)

initFromSink

protected void initFromSink(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess,
                            org.apache.hadoop.mapred.JobConf conf)

initFromTraps

protected void initFromTraps(FlowProcess<org.apache.hadoop.mapred.JobConf> flowProcess,
                             org.apache.hadoop.mapred.JobConf conf)

getTraps

public Set<Tap> getTraps()

getTrap

public Tap getTrap(String name)

getMapperTraps

public Map<String,Tap> getMapperTraps()

getReducerTraps

public Map<String,Tap> getReducerTraps()

getMapperTrap

public Tap getMapperTrap(String name)

getReducerTrap

public Tap getReducerTrap(String name)


Copyright © 2007-2013 Concurrent, Inc. All Rights Reserved.