001 /*
002 * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
003 *
004 * Project and contact information: http://www.cascading.org/
005 *
006 * This file is part of the Cascading project.
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020
021 package cascading.platform;
022
023 import java.io.IOException;
024 import java.util.Comparator;
025 import java.util.HashMap;
026 import java.util.Map;
027
028 import cascading.flow.FlowConnector;
029 import cascading.flow.FlowProcess;
030 import cascading.scheme.Scheme;
031 import cascading.scheme.util.FieldTypeResolver;
032 import cascading.tap.SinkMode;
033 import cascading.tap.Tap;
034 import cascading.tap.partition.Partition;
035 import cascading.tuple.Fields;
036 import org.slf4j.Logger;
037 import org.slf4j.LoggerFactory;
038
039 /**
040 *
041 */
042 public abstract class TestPlatform
043 {
044 private static final Logger LOG = LoggerFactory.getLogger( TestPlatform.class );
045
046 public static final String CLUSTER_TESTING_PROPERTY = "test.cluster.enabled";
047
048 private boolean useCluster = false;
049 private boolean enableCluster = true;
050 private int numMappers = 0;
051 private int numReducers = 0;
052
053 /**
054 * Method getGlobalProperties fetches all "platform." prefixed system properties.
055 * <p/>
056 * Sub-classes of TestPlatform should use these values as overrides before returning from
057 * {@link #getProperties()}.
058 *
059 * @return a Map of properties
060 */
061 public static Map<Object, Object> getGlobalProperties()
062 {
063 HashMap<Object, Object> properties = new HashMap<Object, Object>();
064
065 for( String propertyName : System.getProperties().stringPropertyNames() )
066 {
067 if( propertyName.startsWith( "platform." ) )
068 properties.put( propertyName.substring( "platform.".length() ), System.getProperty( propertyName ) );
069 }
070
071 if( !properties.isEmpty() )
072 LOG.info( "platform property overrides: ", properties );
073
074 return properties;
075 }
076
077 protected TestPlatform()
078 {
079 enableCluster = Boolean.parseBoolean( System.getProperty( CLUSTER_TESTING_PROPERTY, Boolean.toString( enableCluster ) ) );
080 }
081
082 public String getName()
083 {
084 return getClass().getSimpleName().replaceAll( "^(.*)Platform$", "$1" ).toLowerCase();
085 }
086
087 public boolean isMapReduce()
088 {
089 return false;
090 }
091
092 public int getNumMappers()
093 {
094 return numMappers;
095 }
096
097 public void setNumMappers( int numMappers )
098 {
099 this.numMappers = numMappers;
100 }
101
102 public int getNumReducers()
103 {
104 return numReducers;
105 }
106
107 public void setNumReducers( int numReducers )
108 {
109 this.numReducers = numReducers;
110 }
111
112 public void setNumMapTasks( Map<Object, Object> properties, int numMapTasks )
113 {
114 // do nothing
115 }
116
117 public void setNumReduceTasks( Map<Object, Object> properties, int numReduceTasks )
118 {
119 // do nothing
120 }
121
122 public Integer getNumMapTasks( Map<Object, Object> properties )
123 {
124 return null;
125 }
126
127 public Integer getNumReduceTasks( Map<Object, Object> properties )
128 {
129 return null;
130 }
131
132 public abstract void setUp() throws IOException;
133
134 public abstract Map<Object, Object> getProperties();
135
136 public abstract void tearDown();
137
138 public void setUseCluster( boolean useCluster )
139 {
140 this.useCluster = useCluster;
141 }
142
143 public boolean isUseCluster()
144 {
145 return enableCluster && useCluster;
146 }
147
148 public abstract void copyFromLocal( String inputFile ) throws IOException;
149
150 public abstract void copyToLocal( String outputFile ) throws IOException;
151
152 public abstract boolean remoteExists( String outputFile ) throws IOException;
153
154 public abstract boolean remoteRemove( String outputFile, boolean recursive ) throws IOException;
155
156 public abstract FlowProcess getFlowProcess();
157
158 public abstract FlowConnector getFlowConnector( Map<Object, Object> properties );
159
160 public FlowConnector getFlowConnector()
161 {
162 return getFlowConnector( getProperties() );
163 }
164
165 public abstract Tap getTap( Scheme scheme, String filename, SinkMode mode );
166
167 public Tap getTextFile( Fields sourceFields, String filename )
168 {
169 return getTextFile( sourceFields, filename, SinkMode.KEEP );
170 }
171
172 public Tap getTextFile( String filename )
173 {
174 return getTextFile( filename, SinkMode.KEEP );
175 }
176
177 public Tap getTextFile( String filename, SinkMode mode )
178 {
179 return getTextFile( null, filename, mode );
180 }
181
182 public Tap getTextFile( Fields sourceFields, String filename, SinkMode mode )
183 {
184 return getTextFile( sourceFields, Fields.ALL, filename, mode );
185 }
186
187 public abstract Tap getTextFile( Fields sourceFields, Fields sinkFields, String filename, SinkMode mode );
188
189 public Tap getDelimitedFile( Fields fields, String delimiter, String filename )
190 {
191 return getDelimitedFile( fields, false, delimiter, "\"", null, filename, SinkMode.KEEP );
192 }
193
194 public Tap getDelimitedFile( Fields fields, String delimiter, String filename, SinkMode mode )
195 {
196 return getDelimitedFile( fields, false, delimiter, "\"", null, filename, mode );
197 }
198
199 @Deprecated
200 public Tap getDelimitedFile( Fields fields, String filename, SinkMode mode )
201 {
202 return getDelimitedFile( fields, false, "\t", "\"", null, filename, mode );
203 }
204
205 @Deprecated
206 public Tap getDelimitedFile( Fields fields, boolean hasHeader, String filename, SinkMode mode )
207 {
208 return getDelimitedFile( fields, hasHeader, "\t", "\"", null, filename, mode );
209 }
210
211 public Tap getTabDelimitedFile( Fields fields, String filename, SinkMode mode )
212 {
213 return getDelimitedFile( fields, false, "\t", "\"", null, filename, mode );
214 }
215
216 public Tap getTabDelimitedFile( Fields fields, boolean hasHeader, String filename, SinkMode mode )
217 {
218 return getDelimitedFile( fields, hasHeader, "\t", "\"", null, filename, mode );
219 }
220
221 public Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, String filename, SinkMode mode )
222 {
223 return getDelimitedFile( fields, hasHeader, delimiter, quote, null, filename, mode );
224 }
225
226 public Tap getDelimitedFile( Fields fields, String delimiter, String quote, String filename, SinkMode mode )
227 {
228 return getDelimitedFile( fields, false, delimiter, quote, null, filename, mode );
229 }
230
231 public Tap getDelimitedFile( Fields fields, String delimiter, Class[] types, String filename, SinkMode mode )
232 {
233 return getDelimitedFile( fields, false, delimiter, "\"", types, filename, mode );
234 }
235
236 public abstract Tap getDelimitedFile( Fields fields, boolean hasHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
237
238 public abstract Tap getDelimitedFile( Fields fields, boolean skipHeader, boolean writeHeader, String delimiter, String quote, Class[] types, String filename, SinkMode mode );
239
240 public abstract Tap getDelimitedFile( String delimiter, String quote, FieldTypeResolver fieldTypeResolver, String filename, SinkMode mode );
241
242 public abstract Tap getTemplateTap( Tap sink, String pathTemplate, int openThreshold );
243
244 public abstract Tap getTemplateTap( Tap sink, String pathTemplate, Fields fields, int openThreshold );
245
246 public abstract Tap getPartitionTap( Tap sink, Partition partition, int openThreshold );
247
248 public abstract Scheme getTestConfigDefScheme();
249
250 public abstract Scheme getTestFailScheme();
251
252 public abstract Comparator getLongComparator( boolean reverseSort );
253
254 public abstract Comparator getStringComparator( boolean reverseSort );
255
256 public abstract String getHiddenTemporaryPath();
257 }