001 /* 002 * Copyright (c) 2007-2015 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021 package cascading.flow.hadoop2; 022 023 import java.util.Map; 024 import java.util.Properties; 025 import java.util.Set; 026 027 import cascading.flow.hadoop.planner.HadoopPlanner; 028 import cascading.flow.hadoop.util.HadoopUtil; 029 import org.apache.hadoop.conf.Configuration; 030 import org.apache.hadoop.mapred.JobConf; 031 import org.slf4j.Logger; 032 import org.slf4j.LoggerFactory; 033 034 /** 035 * Hadoop 2 (YARN) specific planner implementation. 036 */ 037 public class Hadoop2MR1Planner extends HadoopPlanner 038 { 039 /** Field LOG */ 040 private static final Logger LOG = LoggerFactory.getLogger( Hadoop2MR1Planner.class ); 041 042 /** 043 * Method copyJobConf adds the given JobConf values to the given properties object. Use this method to pass 044 * custom default Hadoop JobConf properties to Hadoop. 045 * 046 * @param properties of type Map 047 * @param configuration of type JobConf 048 */ 049 public static void copyConfiguration( Map<Object, Object> properties, Configuration configuration ) 050 { 051 for( Map.Entry<String, String> entry : configuration ) 052 properties.put( entry.getKey(), entry.getValue() ); 053 } 054 055 /** 056 * Method copyProperties adds the given Map values to the given JobConf object. 057 * 058 * @param configuration of type JobConf 059 * @param properties of type Map 060 */ 061 public static void copyProperties( Configuration configuration, Map<Object, Object> properties ) 062 { 063 if( properties instanceof Properties ) 064 { 065 Properties props = (Properties) properties; 066 Set<String> keys = props.stringPropertyNames(); 067 068 for( String key : keys ) 069 configuration.set( key, props.getProperty( key ) ); 070 } 071 else 072 { 073 for( Map.Entry<Object, Object> entry : properties.entrySet() ) 074 { 075 if( entry.getValue() != null ) 076 configuration.set( entry.getKey().toString(), entry.getValue().toString() ); 077 } 078 } 079 } 080 081 @Override 082 protected void checkPlatform( JobConf jobConf ) 083 { 084 if( !HadoopUtil.isYARN( jobConf ) ) 085 LOG.warn( "running Hadoop 1.x based flows on YARN may cause problems, please use the 'cascading-hadoop' dependencies" ); 086 } 087 }