001/* 002 * Copyright (c) 2016 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading.flow.hadoop.planner; 023 024import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsNodePartitioner; 025import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsStepPartitioner; 026import cascading.flow.hadoop.planner.rule.partitioner.GroupTapNodePartitioner; 027import cascading.flow.hadoop.planner.rule.partitioner.MultiTapGroupNodePartitioner; 028import cascading.flow.hadoop.planner.rule.partitioner.StreamedAccumulatedTapsHashJoinPipelinePartitioner; 029import cascading.flow.hadoop.planner.rule.partitioner.StreamedAccumulatedTapsPipelinePartitioner; 030import cascading.flow.hadoop.planner.rule.partitioner.StreamedOnlySourcesPipelinePartitioner; 031import cascading.flow.hadoop.planner.rule.partitioner.StreamedSelfJoinSourcesPipelinePartitioner; 032import cascading.flow.hadoop.planner.rule.partitioner.TapGroupTapStepPartitioner; 033import cascading.flow.hadoop.planner.rule.transformer.RemoveMalformedHashJoinPipelineTransformer; 034import cascading.flow.hadoop.planner.rule.transformer.ReplaceAccumulateTapWithDistCacheTransformer; 035import cascading.flow.hadoop.planner.rule.transformer.TapBalanceCheckpointTransformer; 036import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupBlockingHashJoinTransformer; 037import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupGroupTransformer; 038import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupMergeGroupTransformer; 039import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupNonBlockingHashJoinTransformer; 040import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitJoinTransformer; 041import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitMergeGroupTransformer; 042import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitMergeTransformer; 043import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitTransformer; 044import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitTriangleTransformer; 045import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinBlockingHashJoinTransformer; 046import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinSameSourceTransformer; 047import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafePipeSplitTransformer; 048import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafeSplitTransformer; 049import cascading.flow.hadoop.planner.rule.transformer.TapBalanceSameSourceStreamedAccumulatedTransformer; 050import cascading.flow.planner.rule.RuleRegistry; 051import cascading.flow.planner.rule.assertion.BufferAfterEveryAssert; 052import cascading.flow.planner.rule.assertion.EveryAfterBufferAssert; 053import cascading.flow.planner.rule.assertion.LoneGroupAssert; 054import cascading.flow.planner.rule.assertion.MissingGroupAssert; 055import cascading.flow.planner.rule.assertion.SplitBeforeEveryAssert; 056import cascading.flow.planner.rule.transformer.ApplyAssertionLevelTransformer; 057import cascading.flow.planner.rule.transformer.ApplyDebugLevelTransformer; 058import cascading.flow.planner.rule.transformer.RemoveNoOpPipeTransformer; 059 060/** 061 * 062 */ 063public class MapReduceHadoopRuleRegistry extends RuleRegistry 064 { 065 public MapReduceHadoopRuleRegistry() 066 { 067// enableDebugLogging(); 068 069 // PreBalance 070 addRule( new LoneGroupAssert() ); 071 addRule( new MissingGroupAssert() ); 072 addRule( new BufferAfterEveryAssert() ); 073 addRule( new EveryAfterBufferAssert() ); 074 addRule( new SplitBeforeEveryAssert() ); 075 076 // Balance with temporary Taps 077 addRule( new TapBalanceGroupSplitTransformer() ); 078 addRule( new TapBalanceGroupSplitJoinTransformer() ); 079 addRule( new TapBalanceGroupSplitTriangleTransformer() ); // testGroupSplitToCoGroupsTriangle 080 addRule( new TapBalanceGroupSplitMergeGroupTransformer() ); 081 addRule( new TapBalanceGroupSplitMergeTransformer() ); 082 addRule( new TapBalanceGroupMergeGroupTransformer() ); 083 addRule( new TapBalanceGroupGroupTransformer() ); 084 addRule( new TapBalanceCheckpointTransformer() ); 085 addRule( new TapBalanceHashJoinSameSourceTransformer() ); 086 addRule( new TapBalanceHashJoinBlockingHashJoinTransformer() ); 087 addRule( new TapBalanceGroupBlockingHashJoinTransformer() ); 088 addRule( new TapBalanceGroupNonBlockingHashJoinTransformer() ); 089 addRule( new TapBalanceSameSourceStreamedAccumulatedTransformer() ); 090 addRule( new TapBalanceNonSafeSplitTransformer() ); 091 addRule( new TapBalanceNonSafePipeSplitTransformer() ); 092 093 // PreResolve 094 addRule( new RemoveNoOpPipeTransformer() ); 095 addRule( new ApplyAssertionLevelTransformer() ); 096 addRule( new ApplyDebugLevelTransformer() ); 097 addRule( new ReplaceAccumulateTapWithDistCacheTransformer() ); 098 099 // PostResolve 100// addRule( new CombineAdjacentTapTransformer() ); 101 102 // PartitionSteps 103 addRule( new ConsecutiveTapsStepPartitioner() ); 104 addRule( new TapGroupTapStepPartitioner() ); 105 106 // PartitionNodes 107 addRule( new ConsecutiveTapsNodePartitioner() ); 108 addRule( new MultiTapGroupNodePartitioner() ); 109 addRule( new GroupTapNodePartitioner() ); 110 111 // PartitionPipelines 112 addRule( new StreamedAccumulatedTapsHashJoinPipelinePartitioner() ); // solves streamed/accumulated annotation hiding 113 addRule( new StreamedAccumulatedTapsPipelinePartitioner() ); 114 addRule( new StreamedSelfJoinSourcesPipelinePartitioner() ); 115 addRule( new StreamedOnlySourcesPipelinePartitioner() ); 116 117 // PostPipelines 118 addRule( new RemoveMalformedHashJoinPipelineTransformer() ); 119 } 120 }