001 /*
002 * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
003 *
004 * Project and contact information: http://www.cascading.org/
005 *
006 * This file is part of the Cascading project.
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020
021 package cascading.tuple.collect;
022
023 import java.util.ArrayList;
024 import java.util.List;
025 import java.util.Properties;
026
027 import cascading.property.Props;
028 import cascading.util.Util;
029
030 /**
031 * Class SpillableProps is a fluent interface for building properties to be passed to a
032 * {@link cascading.flow.FlowConnector} before creating new {@link cascading.flow.Flow} instances.
033 *
034 * @see SpillableTupleList
035 * @see SpillableTupleMap
036 */
037 public class SpillableProps extends Props
038 {
039 /**
040 * Whether to enable compression of the spills or not, on by default.
041 *
042 * @see Boolean#parseBoolean(String)
043 */
044 public static final String SPILL_COMPRESS = "cascading.spill.compress";
045
046 /** A comma delimited list of possible codecs to try. This is platform dependent. */
047 public static final String SPILL_CODECS = "cascading.spill.codecs";
048
049 /** Number of tuples to hold in memory before spilling them to disk. */
050 public static final String LIST_THRESHOLD = "cascading.spill.list.threshold";
051
052 /** The total number of tuple values (not keys) to attempt to keep in memory. */
053 public static final String MAP_THRESHOLD = "cascading.spill.map.threshold";
054
055 /**
056 * The initial hash map capacity.
057 *
058 * @see java.util.HashMap
059 */
060 public static final String MAP_CAPACITY = "cascading.spill.map.capacity";
061
062 /**
063 * The initial hash map load factor.
064 *
065 * @see java.util.HashMap
066 */
067 public static final String MAP_LOADFACTOR = "cascading.spill.map.loadfactor";
068
069 public static final int defaultListThreshold = 10 * 1000;
070
071 public static final int defaultMapThreshold = 10 * 1000;
072 public static final int defaultMapInitialCapacity = 100 * 1000;
073 public static final float defaultMapLoadFactor = 0.75f;
074
075 boolean compressSpill = true;
076 List<String> codecs = new ArrayList<String>();
077
078 int listSpillThreshold = defaultListThreshold;
079
080 int mapSpillThreshold = defaultMapThreshold;
081 int mapInitialCapacity = defaultMapInitialCapacity;
082 float mapLoadFactor = defaultMapLoadFactor;
083
084 /**
085 * Creates a new SpillableProps instance.
086 *
087 * @return SpillableProps instance
088 */
089 public static SpillableProps spillableProps()
090 {
091 return new SpillableProps();
092 }
093
094 public SpillableProps()
095 {
096 }
097
098 public boolean isCompressSpill()
099 {
100 return compressSpill;
101 }
102
103 /**
104 * Method setCompressSpill either enables or disables spill compression. Enabled by default.
105 * <p/>
106 * Spill compression relies on properly configured and available codecs. See {@link #setCodecs(java.util.List)}.
107 *
108 * @param compressSpill type boolean
109 * @return this
110 */
111 public SpillableProps setCompressSpill( boolean compressSpill )
112 {
113 this.compressSpill = compressSpill;
114
115 return this;
116 }
117
118 public List<String> getCodecs()
119 {
120 return codecs;
121 }
122
123 /**
124 * Method setCodecs sets list of possible codec class names to use. They will be loaded in order, if available.
125 * <p/>
126 * This is platform dependent.
127 *
128 * @param codecs type list
129 * @return this
130 */
131 public SpillableProps setCodecs( List<String> codecs )
132 {
133 this.codecs = codecs;
134
135 return this;
136 }
137
138 /**
139 * Method addCodecs adds a list of possible codec class names to use. They will be loaded in order, if available.
140 * <p/>
141 * This is platform dependent.
142 *
143 * @param codecs type list
144 */
145 public SpillableProps addCodecs( List<String> codecs )
146 {
147 this.codecs.addAll( codecs );
148
149 return this;
150 }
151
152 /**
153 * Method addCodec adds a codec class names to use.
154 * <p/>
155 * This is platform dependent.
156 *
157 * @param codec type String
158 */
159 public SpillableProps addCodec( String codec )
160 {
161 this.codecs.add( codec );
162
163 return this;
164 }
165
166 public int getListSpillThreshold()
167 {
168 return listSpillThreshold;
169 }
170
171 /**
172 * Method setListSpillThreshold sets the number of tuples to hold in memory before spilling them to disk.
173 *
174 * @param listSpillThreshold of type int
175 * @return this
176 */
177 public SpillableProps setListSpillThreshold( int listSpillThreshold )
178 {
179 this.listSpillThreshold = listSpillThreshold;
180
181 return this;
182 }
183
184 public int getMapSpillThreshold()
185 {
186 return mapSpillThreshold;
187 }
188
189 /**
190 * Method setMapSpillThreshold the total number of tuple values (not keys) to attempt to keep in memory.
191 * <p/>
192 * The default implementation cannot spill Map keys to disk.
193 *
194 * @param mapSpillThreshold of type int
195 * @return this
196 */
197 public SpillableProps setMapSpillThreshold( int mapSpillThreshold )
198 {
199 this.mapSpillThreshold = mapSpillThreshold;
200
201 return this;
202 }
203
204 public int getMapInitialCapacity()
205 {
206 return mapInitialCapacity;
207 }
208
209 /**
210 * Method setMapInitialCapacity sets the default capacity to be used by the backing Map implementation.
211 *
212 * @param mapInitialCapacity type int
213 * @return this
214 */
215 public SpillableProps setMapInitialCapacity( int mapInitialCapacity )
216 {
217 this.mapInitialCapacity = mapInitialCapacity;
218
219 return this;
220 }
221
222 public float getMapLoadFactor()
223 {
224 return mapLoadFactor;
225 }
226
227 /**
228 * Method setMapLoadFactor sets the default load factor to be used by the backing Map implementation.
229 *
230 * @param mapLoadFactor type float
231 * @return this
232 */
233 public SpillableProps setMapLoadFactor( float mapLoadFactor )
234 {
235 this.mapLoadFactor = mapLoadFactor;
236
237 return this;
238 }
239
240 @Override
241 protected void addPropertiesTo( Properties properties )
242 {
243 for( String codec : codecs )
244 {
245 String codecs = (String) properties.get( SPILL_CODECS );
246
247 properties.put( SPILL_CODECS, Util.join( ",", Util.removeNulls( codecs, codec ) ) );
248 }
249
250 properties.setProperty( SPILL_COMPRESS, Boolean.toString( compressSpill ) );
251 properties.setProperty( LIST_THRESHOLD, Integer.toString( listSpillThreshold ) );
252
253 properties.setProperty( MAP_THRESHOLD, Integer.toString( mapSpillThreshold ) );
254 properties.setProperty( MAP_CAPACITY, Integer.toString( mapInitialCapacity ) );
255 properties.setProperty( MAP_LOADFACTOR, Float.toString( mapLoadFactor ) );
256 }
257 }