001 /*
002 * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved.
003 *
004 * Project and contact information: http://www.cascading.org/
005 *
006 * This file is part of the Cascading project.
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 * http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020
021 package cascading.tuple.collect;
022
023 import java.util.ArrayList;
024 import java.util.List;
025 import java.util.Properties;
026
027 import cascading.property.Props;
028 import cascading.util.Util;
029
030 /**
031 * Class SpillableProps is a fluent interface for building properties to be passed to a
032 * {@link cascading.flow.FlowConnector} before creating new {@link cascading.flow.Flow} instances.
033 *
034 * @see SpillableTupleList
035 * @see SpillableTupleMap
036 */
037 public class SpillableProps extends Props
038 {
039 /**
040 * Whether to enable compress of the spills or not, on by default.
041 *
042 * @see Boolean#parseBoolean(String)
043 */
044 public static final String SPILL_COMPRESS = "cascading.spill.compress";
045
046 /** A comma delimited list of possible codecs to try. This is platform dependent. */
047 public static final String SPILL_CODECS = "cascading.spill.codecs";
048
049 /** Number of tuples to hold in memory before spilling them to disk. */
050 public static final String LIST_THRESHOLD = "cascading.spill.list.threshold";
051
052 /** The total number of tuple values (not keys) to attempt to keep in memory. */
053 public static final String MAP_THRESHOLD = "cascading.spill.map.threshold";
054
055 /**
056 * The initial hash map capacity.
057 *
058 * @see java.util.HashMap
059 */
060 public static final String MAP_CAPACITY = "cascading.spill.map.capacity";
061
062 /**
063 * The initial hash map load factor.
064 *
065 * @see java.util.HashMap
066 */
067 public static final String MAP_LOADFACTOR = "cascading.spill.map.loadfactor";
068
069 public static final int defaultListThreshold = 10 * 1000;
070
071 public static final int defaultMapThreshold = 10 * 1000;
072 public static final int defaultMapInitialCapacity = 100 * 1000;
073 public static final float defaultMapLoadFactor = 0.75f;
074
075 boolean compressSpill = true;
076 List<String> codecs = new ArrayList<String>();
077
078 int listSpillThreshold = defaultListThreshold;
079
080 int mapSpillThreshold = defaultMapThreshold;
081 int mapInitialCapacity = defaultMapInitialCapacity;
082 float mapLoadFactor = defaultMapLoadFactor;
083
084 public static SpillableProps spillableProps()
085 {
086 return new SpillableProps();
087 }
088
089 public SpillableProps()
090 {
091 }
092
093 public boolean isCompressSpill()
094 {
095 return compressSpill;
096 }
097
098 public SpillableProps setCompressSpill( boolean compressSpill )
099 {
100 this.compressSpill = compressSpill;
101
102 return this;
103 }
104
105 public List<String> getCodecs()
106 {
107 return codecs;
108 }
109
110 public SpillableProps setCodecs( List<String> codecs )
111 {
112 this.codecs = codecs;
113
114 return this;
115 }
116
117 public SpillableProps addCodecs( List<String> codecs )
118 {
119 this.codecs.addAll( codecs );
120
121 return this;
122 }
123
124 public SpillableProps addCodec( String codec )
125 {
126 this.codecs.add( codec );
127
128 return this;
129 }
130
131 public int getListSpillThreshold()
132 {
133 return listSpillThreshold;
134 }
135
136 public SpillableProps setListSpillThreshold( int listSpillThreshold )
137 {
138 this.listSpillThreshold = listSpillThreshold;
139
140 return this;
141 }
142
143 public int getMapSpillThreshold()
144 {
145 return mapSpillThreshold;
146 }
147
148 public SpillableProps setMapSpillThreshold( int mapSpillThreshold )
149 {
150 this.mapSpillThreshold = mapSpillThreshold;
151
152 return this;
153 }
154
155 public int getMapInitialCapacity()
156 {
157 return mapInitialCapacity;
158 }
159
160 public SpillableProps setMapInitialCapacity( int mapInitialCapacity )
161 {
162 this.mapInitialCapacity = mapInitialCapacity;
163
164 return this;
165 }
166
167 public float getMapLoadFactor()
168 {
169 return mapLoadFactor;
170 }
171
172 public SpillableProps setMapLoadFactor( float mapLoadFactor )
173 {
174 this.mapLoadFactor = mapLoadFactor;
175
176 return this;
177 }
178
179 @Override
180 protected void addPropertiesTo( Properties properties )
181 {
182 for( String codec : codecs )
183 {
184 String codecs = (String) properties.get( SPILL_CODECS );
185
186 properties.put( SPILL_CODECS, Util.join( ",", Util.removeNulls( codecs, codec ) ) );
187 }
188
189 properties.setProperty( SPILL_COMPRESS, Boolean.toString( compressSpill ) );
190 properties.setProperty( LIST_THRESHOLD, Integer.toString( listSpillThreshold ) );
191
192 properties.setProperty( MAP_THRESHOLD, Integer.toString( mapSpillThreshold ) );
193 properties.setProperty( MAP_CAPACITY, Integer.toString( mapInitialCapacity ) );
194 properties.setProperty( MAP_LOADFACTOR, Float.toString( mapLoadFactor ) );
195 }
196 }