001/* 002 * Copyright (c) 2007-2015 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.flow.hadoop.planner; 022 023import cascading.flow.hadoop.planner.rule.assertion.DualStreamedAccumulatedMergePipelineAssert; 024import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsNodePartitioner; 025import cascading.flow.hadoop.planner.rule.partitioner.ConsecutiveTapsStepPartitioner; 026import cascading.flow.hadoop.planner.rule.partitioner.GroupTapNodePartitioner; 027import cascading.flow.hadoop.planner.rule.partitioner.MultiTapGroupNodePartitioner; 028import cascading.flow.hadoop.planner.rule.partitioner.StreamedAccumulatedTapsPipelinePartitioner; 029import cascading.flow.hadoop.planner.rule.partitioner.StreamedOnlySourcesPipelinePartitioner; 030import cascading.flow.hadoop.planner.rule.partitioner.StreamedSelfJoinSourcesPipelinePartitioner; 031import cascading.flow.hadoop.planner.rule.partitioner.TapGroupTapStepPartitioner; 032import cascading.flow.hadoop.planner.rule.transformer.RemoveMalformedHashJoinPipelineTransformer; 033import cascading.flow.hadoop.planner.rule.transformer.TapBalanceCheckpointTransformer; 034import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupBlockingHashJoinTransformer; 035import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupGroupTransformer; 036import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupMergeGroupTransformer; 037import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupNonBlockingHashJoinTransformer; 038import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitJoinTransformer; 039import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitMergeGroupTransformer; 040import cascading.flow.hadoop.planner.rule.transformer.TapBalanceGroupSplitTransformer; 041import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinBlockingHashJoinTransformer; 042import cascading.flow.hadoop.planner.rule.transformer.TapBalanceHashJoinSameSourceTransformer; 043import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafePipeSplitTransformer; 044import cascading.flow.hadoop.planner.rule.transformer.TapBalanceNonSafeSplitTransformer; 045import cascading.flow.hadoop.planner.rule.transformer.TapBalanceSameSourceStreamedAccumulatedTransformer; 046import cascading.flow.planner.rule.RuleRegistry; 047import cascading.flow.planner.rule.assertion.BufferAfterEveryAssert; 048import cascading.flow.planner.rule.assertion.EveryAfterBufferAssert; 049import cascading.flow.planner.rule.assertion.LoneGroupAssert; 050import cascading.flow.planner.rule.assertion.MissingGroupAssert; 051import cascading.flow.planner.rule.assertion.SplitBeforeEveryAssert; 052import cascading.flow.planner.rule.transformer.ApplyAssertionLevelTransformer; 053import cascading.flow.planner.rule.transformer.ApplyDebugLevelTransformer; 054import cascading.flow.planner.rule.transformer.RemoveNoOpPipeTransformer; 055 056/** 057 * 058 */ 059public class MapReduceHadoopRuleRegistry extends RuleRegistry 060 { 061 public MapReduceHadoopRuleRegistry() 062 { 063// enableDebugLogging(); 064 065 // PreBalance 066 addRule( new LoneGroupAssert() ); 067 addRule( new MissingGroupAssert() ); 068 addRule( new BufferAfterEveryAssert() ); 069 addRule( new EveryAfterBufferAssert() ); 070 addRule( new SplitBeforeEveryAssert() ); 071 072 // Balance with temporary Taps 073 addRule( new TapBalanceGroupSplitTransformer() ); 074 addRule( new TapBalanceGroupSplitJoinTransformer() ); 075 addRule( new TapBalanceGroupSplitMergeGroupTransformer() ); 076 addRule( new TapBalanceGroupMergeGroupTransformer() ); 077 addRule( new TapBalanceGroupGroupTransformer() ); 078 addRule( new TapBalanceCheckpointTransformer() ); 079 addRule( new TapBalanceHashJoinSameSourceTransformer() ); 080 addRule( new TapBalanceHashJoinBlockingHashJoinTransformer() ); 081 addRule( new TapBalanceGroupBlockingHashJoinTransformer() ); 082 addRule( new TapBalanceGroupNonBlockingHashJoinTransformer() ); 083 addRule( new TapBalanceSameSourceStreamedAccumulatedTransformer() ); 084 addRule( new TapBalanceNonSafeSplitTransformer() ); 085 addRule( new TapBalanceNonSafePipeSplitTransformer() ); 086 087 // PreResolve 088 addRule( new RemoveNoOpPipeTransformer() ); 089 addRule( new ApplyAssertionLevelTransformer() ); 090 addRule( new ApplyDebugLevelTransformer() ); 091 092 // PostResolve 093// addRule( new CombineAdjacentTapTransformer() ); 094 095 // PartitionSteps 096 addRule( new ConsecutiveTapsStepPartitioner() ); 097 addRule( new TapGroupTapStepPartitioner() ); 098 099 // PartitionNodes 100 addRule( new ConsecutiveTapsNodePartitioner() ); 101 addRule( new MultiTapGroupNodePartitioner() ); 102 addRule( new GroupTapNodePartitioner() ); 103 104 // PartitionPipelines 105 addRule( new StreamedAccumulatedTapsPipelinePartitioner() ); 106 addRule( new StreamedSelfJoinSourcesPipelinePartitioner() ); 107 addRule( new StreamedOnlySourcesPipelinePartitioner() ); 108 109 // PostPipelines 110 addRule( new RemoveMalformedHashJoinPipelineTransformer() ); 111 112 // remove when GraphFinder supports captured edges 113 addRule( new DualStreamedAccumulatedMergePipelineAssert() ); 114 115 // enable when GraphFinder supports captured edges 116// addRule( new RemoveStreamedBranchTransformer() ); 117 118 } 119 }