001 /* 002 * Copyright (c) 2007-2014 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021 package cascading.flow.stream; 022 023 import java.util.Collection; 024 import java.util.Collections; 025 import java.util.HashSet; 026 import java.util.Iterator; 027 import java.util.Set; 028 029 import cascading.flow.FlowProcess; 030 import cascading.pipe.Splice; 031 import cascading.tuple.Tuple; 032 import cascading.tuple.TupleEntry; 033 import cascading.tuple.Tuples; 034 035 /** 036 * 037 */ 038 public class MemoryCoGroupGate extends MemorySpliceGate 039 { 040 public MemoryCoGroupGate( FlowProcess flowProcess, Splice splice ) 041 { 042 super( flowProcess, splice ); 043 } 044 045 @Override 046 protected boolean isBlockingStreamed() 047 { 048 return true; 049 } 050 051 @Override 052 public void start( Duct previous ) 053 { 054 } 055 056 @Override 057 public void receive( Duct previous, TupleEntry incomingEntry ) 058 { 059 int pos = posMap.get( previous ); 060 061 Tuple valuesTuple = incomingEntry.getTupleCopy(); 062 Tuple groupTuple = keyBuilder[ pos ].makeResult( valuesTuple, null ); // view on valuesTuple 063 064 groupTuple = getDelegatedTuple( groupTuple ); // wrap so hasher/comparator is honored 065 066 keys.add( groupTuple ); 067 keyValues[ pos ].get( groupTuple ).add( valuesTuple ); 068 } 069 070 @Override 071 public void complete( Duct previous ) 072 { 073 if( count.decrementAndGet() != 0 ) 074 return; 075 076 next.start( this ); 077 078 Collection<Tuple>[] collections = new Collection[ orderedPrevious.length ]; 079 Iterator<Tuple> keyIterator = keys.iterator(); 080 081 Set<Tuple> seenNulls = new HashSet<Tuple>(); 082 083 while( keyIterator.hasNext() ) 084 { 085 Tuple keysTuple = keyIterator.next(); 086 087 keyIterator.remove(); 088 089 // provides sql like semantics 090 if( nullsAreNotEqual && Tuples.frequency( keysTuple, null ) != 0 ) 091 { 092 if( seenNulls.contains( keysTuple ) ) 093 continue; 094 095 seenNulls.add( keysTuple ); 096 097 for( int i = 0; i < keyValues.length; i++ ) 098 { 099 Collection<Tuple> values = keyValues[ i ].remove( keysTuple ); 100 101 if( values == null ) 102 continue; 103 104 for( int j = 0; j < keyValues.length; j++ ) 105 collections[ j ] = Collections.EMPTY_LIST; 106 107 collections[ i ] = values; 108 109 push( collections, keysTuple ); 110 } 111 } 112 else 113 { 114 // drain the keys and keyValues collections to preserve memory 115 for( int i = 0; i < keyValues.length; i++ ) 116 { 117 collections[ i ] = keyValues[ i ].remove( keysTuple ); 118 119 if( collections[ i ] == null ) 120 collections[ i ] = Collections.EMPTY_LIST; 121 } 122 123 push( collections, keysTuple ); 124 } 125 } 126 127 keys = createKeySet(); 128 keyValues = createKeyValuesArray(); 129 130 count.set( numIncomingPaths ); 131 132 next.complete( this ); 133 } 134 135 private void push( Collection<Tuple>[] collections, Tuple keysTuple ) 136 { 137 closure.reset( collections ); 138 139 keyEntry.setTuple( closure.getGroupTuple( keysTuple ) ); 140 141 // create Closure type here 142 tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) ); 143 144 next.receive( this, grouping ); 145 } 146 }