001/* 002 * Copyright (c) 2016 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading.flow.planner; 023 024import java.io.Serializable; 025import java.util.HashSet; 026import java.util.LinkedList; 027import java.util.Map; 028import java.util.Set; 029 030import cascading.tuple.Fields; 031 032import static cascading.tuple.Fields.asDeclaration; 033 034/** Class Scope is an internal representation of the linkages between operations. */ 035public class Scope implements Serializable 036 { 037 /** Enum Kind */ 038 public enum Kind 039 { 040 TAP, EACH, EVERY, GROUPBY, COGROUP, HASHJOIN, MERGE 041 } 042 043 /** Field name */ 044 private String name; 045 private LinkedList<String> priorNames = new LinkedList<>(); 046 047 /** Field ordinal */ 048 private Integer ordinal = 0; // do not copy 049 /** Field isStreamed is not accumulated. currently only relevant to HashJoin * */ 050 private boolean isNonBlocking = true; // do not copy 051 052 // copied 053 054 /** Field kind */ 055 private Kind kind; 056 /** Field incomingPassThroughFields */ 057 private Fields incomingPassThroughFields; 058 /** Field remainderPassThroughFields */ 059 private Fields remainderPassThroughFields; 060 061 /** Field argumentSelector */ 062 private Fields operationArgumentFields; 063 /** Field declaredFields */ 064 private Fields operationDeclaredFields; // fields declared by the operation 065 066 /** Field groupingSelectors */ 067 private Map<String, Fields> keySelectors; 068 /** Field sortingSelectors */ 069 private Map<String, Fields> sortingSelectors; 070 071 /** Field outGroupingSelector */ 072 private Fields outGroupingSelector; 073 /** Field outGroupingFields */ 074 private Fields outGroupingFields; // all key fields 075 076 /** Field outValuesSelector */ 077 private Fields outValuesSelector; 078 /** Field outValuesFields */ 079 private Fields outValuesFields; // all value fields, includes keys 080 081 /** Default constructor. */ 082 public Scope() 083 { 084 } 085 086 /** 087 * Copy constructor 088 * 089 * @param scope of type Scope 090 */ 091 public Scope( Scope scope ) 092 { 093 this.name = scope.getName(); 094 copyFields( scope ); 095 } 096 097 /** 098 * Tap constructor 099 * 100 * @param outFields of type Fields 101 */ 102 public Scope( Fields outFields ) 103 { 104 this.kind = Kind.TAP; 105 106 if( outFields == null ) 107 throw new IllegalArgumentException( "fields may not be null" ); 108 109 this.outGroupingFields = outFields; 110 this.outValuesFields = outFields; 111 } 112 113 /** 114 * Constructor Scope creates a new Scope instance. Used by classes Each and Every. 115 * 116 * @param name of type String 117 * @param kind of type Kind 118 * @param incomingPassThroughFields // * @param remainderPassThroughFields of type Fields 119 * @param operationArgumentFields of type Fields 120 * @param operationDeclaredFields of type Fields 121 * @param outGroupingFields of type Fields 122 * @param outValuesFields of type Fields 123 */ 124 public Scope( String name, Kind kind, Fields incomingPassThroughFields, Fields remainderPassThroughFields, Fields operationArgumentFields, Fields operationDeclaredFields, Fields outGroupingFields, Fields outValuesFields ) 125 { 126 this.name = name; 127 this.kind = kind; 128 this.incomingPassThroughFields = incomingPassThroughFields; 129 this.remainderPassThroughFields = remainderPassThroughFields; 130 this.operationArgumentFields = operationArgumentFields; 131 this.operationDeclaredFields = operationDeclaredFields; 132 133 if( outGroupingFields == null ) 134 throw new IllegalArgumentException( "grouping may not be null" ); 135 136 if( outValuesFields == null ) 137 throw new IllegalArgumentException( "values may not be null" ); 138 139 if( kind == Kind.EACH || kind == Kind.EVERY ) 140 { 141 this.outGroupingSelector = outGroupingFields; 142 this.outGroupingFields = asDeclaration( outGroupingFields ); 143 this.outValuesSelector = outValuesFields; 144 this.outValuesFields = asDeclaration( outValuesFields ); 145 } 146 else 147 { 148 throw new IllegalArgumentException( "may not use the constructor for kind: " + kind ); 149 } 150 } 151 152 /** 153 * Constructor Scope creates a new Scope instance. Used by the Group class. 154 * 155 * @param name of type String 156 * @param operationDeclaredFields of type Fields 157 * @param outGroupingFields of type Fields 158 * @param keySelectors of type Map<String, Fields> 159 * @param sortingSelectors of type Fields 160 * @param outValuesFields of type Fields 161 * @param kind of type boolean 162 */ 163 public Scope( String name, Fields operationDeclaredFields, Fields outGroupingFields, Map<String, Fields> keySelectors, Map<String, Fields> sortingSelectors, Fields outValuesFields, Kind kind ) 164 { 165 this.name = name; 166 this.kind = kind; 167 168 if( keySelectors == null ) 169 throw new IllegalArgumentException( "grouping may not be null" ); 170 171 if( outValuesFields == null ) 172 throw new IllegalArgumentException( "values may not be null" ); 173 174 this.operationDeclaredFields = operationDeclaredFields; 175 this.outGroupingFields = asDeclaration( outGroupingFields ); 176 this.keySelectors = keySelectors; 177 this.sortingSelectors = sortingSelectors; // null ok 178 this.outValuesFields = asDeclaration( outValuesFields ); 179 } 180 181 /** 182 * Constructor Scope creates a new Scope instance. 183 * 184 * @param name of type String 185 */ 186 public Scope( String name ) 187 { 188 this.name = name; 189 } 190 191 public void addPriorNames( Scope incoming, Scope outgoing ) 192 { 193 priorNames.addAll( 0, outgoing.priorNames ); 194 priorNames.addFirst( incoming.getName() ); 195 priorNames.addAll( 0, incoming.priorNames ); 196 } 197 198 public String getPriorName() 199 { 200 if( priorNames.isEmpty() ) 201 return getName(); 202 203 return priorNames.getFirst(); 204 } 205 206 public Integer getOrdinal() 207 { 208 return ordinal; 209 } 210 211 public void setOrdinal( Integer ordinal ) 212 { 213 this.ordinal = ordinal; 214 } 215 216 public void setNonBlocking( boolean isNonBlocking ) 217 { 218 this.isNonBlocking = isNonBlocking; 219 } 220 221 public boolean isNonBlocking() 222 { 223 return isNonBlocking; 224 } 225 226 public boolean isSplice() 227 { 228 return isGroupBy() || isCoGroup() || isMerge() || isHashJoin(); 229 } 230 231 public boolean isGroup() 232 { 233 return kind == Kind.GROUPBY || kind == Kind.COGROUP; 234 } 235 236 public boolean isGroupBy() 237 { 238 return kind == Kind.GROUPBY; 239 } 240 241 public boolean isCoGroup() 242 { 243 return kind == Kind.COGROUP; 244 } 245 246 public boolean isMerge() 247 { 248 return kind == Kind.MERGE; 249 } 250 251 public boolean isHashJoin() 252 { 253 return kind == Kind.HASHJOIN; 254 } 255 256 /** 257 * Method isEach returns true if this Scope object represents an Each. 258 * 259 * @return the each (type boolean) of this Scope object. 260 */ 261 public boolean isEach() 262 { 263 return kind == Kind.EACH; 264 } 265 266 /** 267 * Method isEvery returns true if this Scope object represents an Every. 268 * 269 * @return the every (type boolean) of this Scope object. 270 */ 271 public boolean isEvery() 272 { 273 return kind == Kind.EVERY; 274 } 275 276 /** 277 * Method isTap returns true if this Scope object represents a Tap. 278 * 279 * @return the tap (type boolean) of this Scope object. 280 */ 281 public boolean isTap() 282 { 283 return kind == Kind.TAP; 284 } 285 286 /** 287 * Method getName returns the name of this Scope object. 288 * 289 * @return the name (type String) of this Scope object. 290 */ 291 public String getName() 292 { 293 return name; 294 } 295 296 /** 297 * Method setName sets the name of this Scope object. 298 * 299 * @param name the name of this Scope object. 300 */ 301 public void setName( String name ) 302 { 303 this.name = name; 304 } 305 306 /** 307 * Method getRemainderFields returns the remainderFields of this Scope object. 308 * 309 * @return the remainderFields (type Fields) of this Scope object. 310 */ 311 public Fields getRemainderPassThroughFields() 312 { 313 return remainderPassThroughFields; 314 } 315 316 /** 317 * Method getArgumentSelector returns the argumentSelector of this Scope object. 318 * 319 * @return the argumentSelector (type Fields) of this Scope object. 320 */ 321 public Fields getArgumentsSelector() 322 { 323 return operationArgumentFields; 324 } 325 326 /** 327 * Method getArguments returns the arguments of this Scope object. 328 * 329 * @return the arguments (type Fields) of this Scope object. 330 */ 331 public Fields getArgumentsDeclarator() 332 { 333 return asDeclaration( operationArgumentFields ); 334 } 335 336 /** 337 * Method getDeclaredFields returns the declaredFields of this Scope object. 338 * 339 * @return the declaredFields (type Fields) of this Scope object. 340 */ 341 public Fields getOperationDeclaredFields() 342 { 343 return operationDeclaredFields; 344 } 345 346 /** 347 * Method getGroupingSelectors returns the groupingSelectors of this Scope object. 348 * 349 * @return the groupingSelectors (type Map<String, Fields>) of this Scope object. 350 */ 351 public Map<String, Fields> getKeySelectors() 352 { 353 return keySelectors; 354 } 355 356 /** 357 * Method getSortingSelectors returns the sortingSelectors of this Scope object. 358 * 359 * @return the sortingSelectors (type Map<String, Fields>) of this Scope object. 360 */ 361 public Map<String, Fields> getSortingSelectors() 362 { 363 return sortingSelectors; 364 } 365 366 /** 367 * Method getOutGroupingSelector returns the outGroupingSelector of this Scope object. 368 * 369 * @return the outGroupingSelector (type Fields) of this Scope object. 370 */ 371 public Fields getOutGroupingSelector() 372 { 373 return outGroupingSelector; 374 } 375 376 public Fields getIncomingTapFields() 377 { 378 if( isEvery() ) 379 return getOutGroupingFields(); 380 else 381 return getOutValuesFields(); 382 } 383 384 public Fields getIncomingFunctionArgumentFields() 385 { 386 if( isEvery() ) 387 return getOutGroupingFields(); 388 else 389 return getOutValuesFields(); 390 } 391 392 public Fields getIncomingFunctionPassThroughFields() 393 { 394 if( isEvery() ) 395 return getOutGroupingFields(); 396 else 397 return getOutValuesFields(); 398 } 399 400 public Fields getIncomingAggregatorArgumentFields() 401 { 402 if( isEach() || isTap() ) 403 throw new IllegalStateException( "Every cannot follow a Tap or an Each" ); 404 405 return getOutValuesFields(); 406 } 407 408 public Fields getIncomingAggregatorPassThroughFields() 409 { 410 if( isEach() || isTap() ) 411 throw new IllegalStateException( "Every cannot follow a Tap or an Each" ); 412 413 return getOutGroupingFields(); 414 } 415 416 public Fields getIncomingBufferArgumentFields() 417 { 418 if( isEach() || isTap() ) 419 throw new IllegalStateException( "Every cannot follow a Tap or an Each" ); 420 421 return getOutValuesFields(); 422 } 423 424 public Fields getIncomingBufferPassThroughFields() 425 { 426 if( isEach() || isTap() ) 427 throw new IllegalStateException( "Every cannot follow a Tap or an Each" ); 428 429 return getOutValuesFields(); 430 } 431 432 public Fields getIncomingSpliceFields() 433 { 434 if( isEvery() ) 435 return getOutGroupingFields(); 436 else 437 return getOutValuesFields(); 438 } 439 440 /** 441 * Method getOutGroupingFields returns the outGroupingFields of this Scope object. 442 * 443 * @return the outGroupingFields (type Fields) of this Scope object. 444 */ 445 public Fields getOutGroupingFields() 446 { 447 if( !isSplice() ) 448 return outGroupingFields; 449 450 Fields first = keySelectors.values().iterator().next(); 451 452 // if more than one, this is a merge, so same key names are expected 453 if( keySelectors.size() == 1 || isGroupBy() ) 454 return first; 455 456 // handling CoGroup only 457 458 // if given by user as resultGroupFields 459 if( outGroupingFields != null ) 460 return outGroupingFields; 461 462 // todo throw an exception if we make it this far 463 464 // if all have the same names, then use for grouping 465 Set<Fields> set = new HashSet<Fields>( keySelectors.values() ); 466 467 if( set.size() == 1 ) 468 return first; 469 470 return Fields.size( first.size() ); 471 } 472 473 public Fields getOutGroupingValueFields() 474 { 475 return getOutValuesFields().subtract( getOutGroupingFields() ); 476 } 477 478 /** 479 * Method getOutValuesSelector returns the outValuesSelector of this Scope object. 480 * 481 * @return the outValuesSelector (type Fields) of this Scope object. 482 */ 483 public Fields getOutValuesSelector() 484 { 485 return outValuesSelector; 486 } 487 488 /** 489 * Method getOutValuesFields returns the outValuesFields of this Scope object. 490 * 491 * @return the outValuesFields (type Fields) of this Scope object. 492 */ 493 public Fields getOutValuesFields() 494 { 495 return outValuesFields; 496 } 497 498 /** 499 * Method copyFields copies the given Scope instance fields to this instance. 500 * 501 * @param scope of type Scope 502 */ 503 public void copyFields( Scope scope ) 504 { 505 this.kind = scope.kind; 506 this.incomingPassThroughFields = scope.incomingPassThroughFields; 507 this.remainderPassThroughFields = scope.remainderPassThroughFields; 508 this.operationArgumentFields = scope.operationArgumentFields; 509 this.operationDeclaredFields = scope.operationDeclaredFields; 510 this.keySelectors = scope.keySelectors; 511 this.sortingSelectors = scope.sortingSelectors; 512 this.outGroupingSelector = scope.outGroupingSelector; 513 this.outGroupingFields = scope.outGroupingFields; 514 this.outValuesSelector = scope.outValuesSelector; 515 this.outValuesFields = scope.outValuesFields; 516 } 517 518 public String printSimple() 519 { 520 StringBuilder buffer = new StringBuilder(); 521 522 if( name != null ) 523 buffer.append( "[" ).append( name ).append( "]" ); 524 525 buffer.append( "{id=" ).append( System.identityHashCode( this ) ); 526 527 if( ordinal != null ) 528 buffer.append( ",ordinal=" ).append( ordinal ); 529 530 if( kind != null ) 531 buffer.append( ",kind=" ).append( kind ); 532 533 buffer.append( "}" ); 534 535 return buffer.toString(); 536 } 537 538 @Override 539 public String toString() 540 { 541 return print(); 542 } 543 544 public String print() 545 { 546 StringBuilder buffer = new StringBuilder(); 547 548 if( ordinal != null ) 549 buffer.append( "[" ).append( ordinal ).append( "]" ); 550 551 if( getOutValuesFields() == null ) 552 return buffer.toString(); // endpipes 553 554 buffer.append( "\n" ); 555 556 if( keySelectors != null && !keySelectors.isEmpty() ) 557 { 558 for( String name : keySelectors.keySet() ) 559 { 560 if( buffer.length() != 0 && buffer.charAt( buffer.length() - 1 ) != '\n' ) 561 buffer.append( " | " ); 562 563 buffer.append( name ).append( keySelectors.get( name ).printVerbose() ); 564 } 565 566 buffer.append( "\n" ); 567 } 568 569 if( outGroupingFields != null ) 570 buffer.append( getOutGroupingFields().printVerbose() ).append( "\n" ); 571 572 buffer.append( getOutValuesFields().printVerbose() ); 573 574 return buffer.toString(); 575 } 576 }