001/* 002 * Copyright (c) 2007-2016 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.flow.hadoop; 022 023import cascading.flow.FlowNode; 024import cascading.flow.FlowProcess; 025import cascading.flow.hadoop.util.HadoopMRUtil; 026import cascading.flow.planner.BaseFlowNode; 027import cascading.flow.planner.graph.BaseElementGraph; 028import cascading.flow.planner.graph.ElementGraph; 029import cascading.flow.planner.process.FlowNodeGraph; 030import cascading.flow.planner.process.ProcessEdge; 031import cascading.tap.Tap; 032import org.apache.hadoop.mapred.JobConf; 033 034/** Class MapReduceFlowStep wraps a {@link JobConf} and allows it to be executed as a {@link cascading.flow.Flow}. */ 035public class MapReduceFlowStep extends HadoopFlowStep 036 { 037 /** Field jobConf */ 038 private final JobConf jobConf; 039 040 protected MapReduceFlowStep( String flowName, String stepName, JobConf jobConf, Tap sink ) 041 { 042 this( flowName, stepName, BaseElementGraph.NULL, createFlowNodeGraph( jobConf ), jobConf ); 043 044 addSink( "default", sink ); 045 } 046 047 protected MapReduceFlowStep( String flowName, String stepName, ElementGraph elementGraph, FlowNodeGraph flowNodeGraph, JobConf jobConf ) 048 { 049 super( elementGraph, flowNodeGraph ); 050 setName( stepName ); 051 setFlowName( flowName ); 052 this.jobConf = jobConf; 053 } 054 055 @Override 056 public JobConf createInitializedConfig( FlowProcess<JobConf> flowProcess, JobConf parentConfig ) 057 { 058 return jobConf; 059 } 060 061 protected static FlowNodeGraph createFlowNodeGraph( JobConf jobConf ) 062 { 063 FlowNodeGraph flowNodeGraph = new FlowNodeGraph(); 064 int nodes = 1; 065 boolean hasReducer = HadoopMRUtil.hasReducer( jobConf ); 066 067 if( hasReducer ) 068 nodes = 2; 069 070 FlowNode mapperNode = new BaseFlowNode( String.format( "(1/%s)", nodes ), 0 ); 071 flowNodeGraph.addVertex( mapperNode ); 072 073 if( hasReducer ) 074 { 075 FlowNode reducerNode = new BaseFlowNode( "(2/2)", 1 ); 076 flowNodeGraph.addVertex( reducerNode ); 077 flowNodeGraph.addEdge( mapperNode, reducerNode, new ProcessEdge( mapperNode, reducerNode ) ); 078 } 079 080 return flowNodeGraph; 081 } 082 }