001/* 002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.flow.hadoop2; 022 023import java.beans.ConstructorProperties; 024import java.util.Map; 025 026import cascading.flow.FlowConnector; 027import cascading.flow.hadoop.planner.MapReduceHadoopRuleRegistry; 028import cascading.flow.planner.FlowPlanner; 029import cascading.flow.planner.rule.RuleRegistrySet; 030import cascading.scheme.Scheme; 031import cascading.scheme.hadoop.SequenceFile; 032 033/** 034 * Use the Hadoop2MR1FlowConnector to link source and sink {@link cascading.tap.Tap} instances with an assembly of {@link cascading.pipe.Pipe} instances into 035 * an executable {@link cascading.flow.hadoop.HadoopFlow} for execution on an Apache Hadoop cluster. 036 * 037 * @see cascading.property.AppProps 038 * @see cascading.flow.FlowConnectorProps 039 * @see cascading.flow.FlowDef 040 * @see cascading.flow.hadoop.MapReduceFlow 041 */ 042public class Hadoop2MR1FlowConnector extends FlowConnector 043 { 044 /** 045 * Constructor FlowConnector creates a new FlowConnector instance. 046 * <p/> 047 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 048 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 049 */ 050 public Hadoop2MR1FlowConnector() 051 { 052 } 053 054 /** 055 * Constructor FlowConnector creates a new FlowConnector instance using the given {@link java.util.Properties} instance as 056 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 057 * 058 * @param properties of type Properties 059 */ 060 @ConstructorProperties({"properties"}) 061 public Hadoop2MR1FlowConnector( Map<Object, Object> properties ) 062 { 063 super( properties ); 064 } 065 066 /** 067 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance. 068 * <p/> 069 * All properties passed to Hadoop are retrieved from a default instantiation of the Hadoop 070 * {@link org.apache.hadoop.mapred.JobConf} which pulls all properties from the local CLASSPATH. 071 * 072 * @param ruleRegistrySet of type RuleRegistry 073 */ 074 @ConstructorProperties({"ruleRegistrySet"}) 075 public Hadoop2MR1FlowConnector( RuleRegistrySet ruleRegistrySet ) 076 { 077 super( ruleRegistrySet ); 078 } 079 080 /** 081 * Constructor HadoopFlowConnector creates a new HadoopFlowConnector instance using the given {@link java.util.Properties} instance as 082 * default value for the underlying jobs. All properties are copied to a new native configuration instance. 083 * 084 * @param properties of type Map 085 * @param ruleRegistrySet of type RuleRegistry 086 */ 087 @ConstructorProperties({"properties", "ruleRegistrySet"}) 088 public Hadoop2MR1FlowConnector( Map<Object, Object> properties, RuleRegistrySet ruleRegistrySet ) 089 { 090 super( properties, ruleRegistrySet ); 091 } 092 093 @Override 094 protected Class<? extends Scheme> getDefaultIntermediateSchemeClass() 095 { 096 return SequenceFile.class; 097 } 098 099 @Override 100 protected FlowPlanner createFlowPlanner() 101 { 102 return new Hadoop2MR1Planner(); 103 } 104 105 @Override 106 protected RuleRegistrySet createDefaultRuleRegistrySet() 107 { 108 return new RuleRegistrySet( new MapReduceHadoopRuleRegistry() ); 109 } 110 }