public class HadoopPlanner extends cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
FlowConnector
sub-class.
Notes:
Custom JobConf propertiescopyJobConf(java.util.Map, org.apache.hadoop.mapred.JobConf)
on a map properties object before constructing a new FlowConnector
sub-class.
A better practice would be to set Hadoop properties directly on the map properties object handed to the FlowConnector.
All values in the map will be passed to a new default JobConf instance to be used as defaults for all resulting
Flow instances.
For example, properties.set("mapred.child.java.opts","-Xmx512m");
would convince Hadoop
to spawn all child jvms with a heap of 512MB.Modifier and Type | Field and Description |
---|---|
static java.lang.String |
PLATFORM_NAME |
Constructor and Description |
---|
HadoopPlanner() |
Modifier and Type | Method and Description |
---|---|
protected void |
checkPlatform(Configuration conf) |
void |
configRuleRegistryDefaults(cascading.flow.planner.rule.RuleRegistry ruleRegistry) |
static void |
copyJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
JobConf jobConf)
Method copyJobConf adds the given JobConf values to the given properties object.
|
static void |
copyProperties(JobConf jobConf,
java.util.Map<java.lang.Object,java.lang.Object> properties)
Method copyProperties adds the given Map values to the given JobConf object.
|
protected HadoopFlow |
createFlow(cascading.flow.FlowDef flowDef) |
static JobConf |
createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties)
Method createJobConf returns a new JobConf instance using the values in the given properties argument.
|
JobConf |
getDefaultConfig() |
java.net.URI |
getDefaultURIScheme(cascading.tap.Tap tap) |
cascading.flow.planner.process.FlowStepFactory<JobConf> |
getFlowStepFactory() |
cascading.flow.planner.PlannerInfo |
getPlannerInfo(java.lang.String registryName) |
cascading.flow.planner.PlatformInfo |
getPlatformInfo() |
java.net.URI |
getURIScheme(cascading.tap.Tap tap) |
void |
initialize(cascading.flow.FlowConnector flowConnector,
java.util.Map<java.lang.Object,java.lang.Object> properties) |
protected cascading.tap.Tap |
makeTempTap(java.lang.String prefix,
java.lang.String name) |
buildFlow, createFlowElementGraph, getAssertionLevel, getDebugLevel, getDefaultProperties, getFlowNodeFactory, handleExceptionDuringPlanning, makeTempTap, resolveAssemblyPlanners, resolveTails, verifyAllTaps, verifyAssembly, verifyCheckpoints, verifyPipeAssemblyEndPoints, verifyResult, verifyResultInternal, verifySourceNotSinks, verifyTaps, verifyTraps
public static final java.lang.String PLATFORM_NAME
public HadoopPlanner()
public static void copyJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, JobConf jobConf)
properties
- of type MapjobConf
- of type JobConfpublic static JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties)
properties
- of type Mappublic static void copyProperties(JobConf jobConf, java.util.Map<java.lang.Object,java.lang.Object> properties)
jobConf
- of type JobConfproperties
- of type Mappublic cascading.flow.planner.PlannerInfo getPlannerInfo(java.lang.String registryName)
getPlannerInfo
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public JobConf getDefaultConfig()
getDefaultConfig
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public cascading.flow.planner.PlatformInfo getPlatformInfo()
getPlatformInfo
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public void initialize(cascading.flow.FlowConnector flowConnector, java.util.Map<java.lang.Object,java.lang.Object> properties)
initialize
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public void configRuleRegistryDefaults(cascading.flow.planner.rule.RuleRegistry ruleRegistry)
configRuleRegistryDefaults
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
protected void checkPlatform(Configuration conf)
protected HadoopFlow createFlow(cascading.flow.FlowDef flowDef)
createFlow
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public cascading.flow.planner.process.FlowStepFactory<JobConf> getFlowStepFactory()
getFlowStepFactory
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
public java.net.URI getDefaultURIScheme(cascading.tap.Tap tap)
public java.net.URI getURIScheme(cascading.tap.Tap tap)
protected cascading.tap.Tap makeTempTap(java.lang.String prefix, java.lang.String name)
makeTempTap
in class cascading.flow.planner.FlowPlanner<HadoopFlow,JobConf>
Copyright © 2007-2015 Xplenty, Inc. All Rights Reserved.