001 /* 002 * Copyright (c) 2007-2015 Concurrent, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021 package cascading.stats.hadoop; 022 023 import java.util.HashMap; 024 import java.util.Map; 025 026 import cascading.stats.CascadingStats; 027 import org.apache.hadoop.mapred.Counters; 028 import org.apache.hadoop.mapred.TaskCompletionEvent; 029 import org.apache.hadoop.mapred.TaskReport; 030 031 import static cascading.stats.CascadingStats.Status.*; 032 033 /** Class HadoopTaskStats tracks individual task stats. */ 034 public class HadoopSliceStats 035 { 036 private final CascadingStats.Status parentStatus; 037 038 public static class HadoopAttempt 039 { 040 private final TaskCompletionEvent event; 041 042 public HadoopAttempt( TaskCompletionEvent event ) 043 { 044 this.event = event; 045 } 046 047 public int getEventId() 048 { 049 return event.getEventId(); 050 } 051 052 public int getTaskRunTime() 053 { 054 return event.getTaskRunTime(); 055 } 056 057 public String getTaskStatus() 058 { 059 return event.getTaskStatus().toString(); 060 } 061 062 public String getTaskTrackerHttp() 063 { 064 return event.getTaskTrackerHttp(); 065 } 066 067 public CascadingStats.Status getStatusFor() 068 { 069 CascadingStats.Status status = null; 070 071 switch( event.getTaskStatus() ) 072 { 073 case FAILED: 074 status = FAILED; 075 break; 076 case KILLED: 077 status = STOPPED; 078 break; 079 case SUCCEEDED: 080 status = SUCCESSFUL; 081 break; 082 case OBSOLETE: 083 status = SKIPPED; 084 break; 085 case TIPFAILED: 086 status = FAILED; 087 break; 088 } 089 return status; 090 } 091 } 092 093 public enum Kind 094 { 095 SETUP, MAPPER, REDUCER, CLEANUP 096 } 097 098 private String id; 099 private Kind kind; 100 private final boolean parentStepHasReducers; 101 private TaskReport taskReport; 102 private Map<String, Map<String, Long>> counters; 103 104 private Map<Integer, HadoopAttempt> attempts = new HashMap<Integer, HadoopAttempt>(); 105 106 HadoopSliceStats( String id, CascadingStats.Status parentStatus, Kind kind, boolean parentStepHasReducers, TaskReport taskReport ) 107 { 108 this.parentStatus = parentStatus; 109 this.id = id; 110 this.kind = kind; 111 this.parentStepHasReducers = parentStepHasReducers; 112 this.taskReport = taskReport; 113 } 114 115 public String getID() 116 { 117 return id; 118 } 119 120 public Kind getKind() 121 { 122 return kind; 123 } 124 125 /** 126 * Method getId returns the Hadoop task id. 127 * 128 * @return the id (type String) of this HadoopTaskStats object. 129 */ 130 public String getTaskID() 131 { 132 return taskReport.getTaskID().toString(); 133 } 134 135 public int getTaskIDNum() 136 { 137 return taskReport.getTaskID().getId(); 138 } 139 140 public String getJobID() 141 { 142 return taskReport.getTaskID().getJobID().toString(); 143 } 144 145 public boolean parentStepHasReducers() 146 { 147 return parentStepHasReducers; 148 } 149 150 public float getProgress() 151 { 152 return taskReport.getProgress(); 153 } 154 155 public String getState() 156 { 157 return taskReport.getState(); 158 } 159 160 public long getStartTime() 161 { 162 return taskReport.getStartTime(); 163 } 164 165 public long getFinishTime() 166 { 167 return taskReport.getFinishTime(); 168 } 169 170 public CascadingStats.Status getParentStatus() 171 { 172 return parentStatus; 173 } 174 175 public CascadingStats.Status getStatus() 176 { 177 CascadingStats.Status status = null; 178 179 switch( taskReport.getCurrentStatus() ) 180 { 181 case PENDING: 182 status = PENDING; 183 break; 184 case RUNNING: 185 status = RUNNING; 186 break; 187 case COMPLETE: 188 status = SUCCESSFUL; 189 break; 190 case KILLED: 191 status = STOPPED; 192 break; 193 case FAILED: 194 status = FAILED; 195 break; 196 } 197 198 return status; 199 } 200 201 public String[] getDiagnostics() 202 { 203 return taskReport.getDiagnostics(); 204 } 205 206 public Map<String, Map<String, Long>> getCounters() 207 { 208 if( counters == null ) 209 setCounters( taskReport ); 210 211 return counters; 212 } 213 214 public Map<Integer, HadoopAttempt> getAttempts() 215 { 216 return attempts; 217 } 218 219 private void setCounters( TaskReport taskReport ) 220 { 221 this.counters = new HashMap<String, Map<String, Long>>(); 222 223 Counters hadoopCounters = taskReport.getCounters(); 224 225 for( Counters.Group group : hadoopCounters ) 226 { 227 Map<String, Long> values = new HashMap<String, Long>(); 228 229 this.counters.put( group.getName(), values ); 230 231 for( Counters.Counter counter : group ) 232 values.put( counter.getName(), counter.getCounter() ); 233 } 234 } 235 236 /** 237 * Method getCounterValue returns the raw Hadoop counter value. 238 * 239 * @param counter of Enum 240 * @return long 241 */ 242 public long getCounterValue( Enum counter ) 243 { 244 return getCounterValue( counter.getDeclaringClass().getName(), counter.name() ); 245 } 246 247 /** 248 * Method getCounterValue returns the raw Hadoop counter value. 249 * 250 * @param group of String 251 * @param name of String 252 * @return long 253 */ 254 public long getCounterValue( String group, String name ) 255 { 256 if( getCounters() == null || getCounters().get( group ) == null ) 257 return 0; 258 259 Long value = getCounters().get( group ).get( name ); 260 261 if( value == null ) 262 return 0; 263 264 return value; 265 } 266 267 public void addAttempt( TaskCompletionEvent event ) 268 { 269 attempts.put( event.getEventId(), new HadoopAttempt( event ) ); 270 } 271 272 @Override 273 public String toString() 274 { 275 final StringBuilder sb = new StringBuilder(); 276 sb.append( "HadoopTaskStats" ); 277 sb.append( "{id='" ).append( id ).append( '\'' ); 278 sb.append( ", kind=" ).append( kind ); 279 sb.append( '}' ); 280 return sb.toString(); 281 } 282 }