001/* 002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 003 * 004 * Project and contact information: http://www.cascading.org/ 005 * 006 * This file is part of the Cascading project. 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020 021package cascading.stats.tez.util; 022 023import java.io.IOException; 024import java.util.EnumSet; 025import java.util.Set; 026 027import cascading.CascadingException; 028import cascading.flow.hadoop.util.HadoopUtil; 029import cascading.flow.planner.PlatformInfo; 030import cascading.util.Util; 031import org.apache.hadoop.yarn.api.records.ApplicationId; 032import org.apache.hadoop.yarn.api.records.ApplicationReport; 033import org.apache.hadoop.yarn.conf.YarnConfiguration; 034import org.apache.hadoop.yarn.exceptions.YarnException; 035import org.apache.tez.client.FrameworkClient; 036import org.apache.tez.client.TezClient; 037import org.apache.tez.dag.api.DAG; 038import org.apache.tez.dag.api.TezConfiguration; 039import org.apache.tez.dag.api.TezException; 040import org.apache.tez.dag.api.client.DAGClient; 041import org.apache.tez.dag.api.client.DAGStatus; 042import org.apache.tez.dag.api.client.StatusGetOpts; 043import org.slf4j.Logger; 044import org.slf4j.LoggerFactory; 045 046/** 047 * 048 */ 049public class TezStatsUtil 050 { 051 private static final Logger LOG = LoggerFactory.getLogger( TezStatsUtil.class ); 052 053 public static final Set<StatusGetOpts> STATUS_GET_COUNTERS = EnumSet.of( StatusGetOpts.GET_COUNTERS ); 054 055 static Class<DAGClient> timelineClientClass = null; 056 057 public static final String TIMELINE_CLIENT_CLASS = "cascading.stats.tez.util.TezTimelineClient"; 058 059 public static String getPlatformVersion() 060 { 061 PlatformInfo tez = HadoopUtil.getPlatformInfo( DAG.class, null, "Tez" ); 062 063 if( tez == null || tez.version == null ) 064 return "unknown"; 065 066 return tez.version; 067 } 068 069 private static boolean loadClass() 070 { 071 if( timelineClientClass != null ) 072 return true; 073 074 try 075 { 076 timelineClientClass = (Class<DAGClient>) Thread.currentThread().getContextClassLoader().loadClass( TIMELINE_CLIENT_CLASS ); 077 078 return true; 079 } 080 catch( ClassNotFoundException exception ) 081 { 082 LOG.error( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is enabled, yet unable to load Tez YARN timeline client class: {}, ensure these dependencies are in your local CLASSPATH: tez-yarn-timeline-history, org.apache.tez:tez-yarn-timeline-history or org.apache.tez:tez-yarn-timeline-history-with-acls", TIMELINE_CLIENT_CLASS, exception ); 083 } 084 085 return false; 086 } 087 088 public static DAGStatus getDagStatusWithCounters( DAGClient dagClient ) 089 { 090 if( dagClient == null ) 091 return null; 092 093 try 094 { 095 return dagClient.getDAGStatus( STATUS_GET_COUNTERS ); 096 } 097 catch( IOException | TezException exception ) 098 { 099 throw new CascadingException( "unable to get counters from dag client", exception ); 100 } 101 } 102 103 /** 104 * Only called if the service is enabled 105 * 106 * @param dagClient 107 * @return 108 */ 109 public static DAGClient createTimelineClient( DAGClient dagClient ) 110 { 111 if( dagClient == null ) 112 return null; 113 114 if( !loadClass() ) 115 return null; 116 117 // TezTimelineClient( ApplicationId appId, String dagId, TezConfiguration conf, FrameworkClient frameworkClient, DAGClient dagClient ) 118 119 Class[] types = new Class[]{ 120 ApplicationId.class, 121 String.class, 122 TezConfiguration.class, 123 FrameworkClient.class, 124 DAGClient.class 125 }; 126 127 ApplicationId appId = Util.returnInstanceFieldIfExistsSafe( dagClient, "appId" ); 128 String dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); 129 TezConfiguration conf = Util.returnInstanceFieldIfExistsSafe( dagClient, "conf" ); 130 FrameworkClient frameworkClient = Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" ); 131 132 Object[] parameters = new Object[]{ 133 appId, 134 dagId, 135 conf, 136 frameworkClient, 137 dagClient 138 }; 139 140 try 141 { 142 return Util.invokeConstructor( timelineClientClass, parameters, types ); 143 } 144 catch( CascadingException exception ) 145 { 146 Throwable cause = exception.getCause(); 147 148 if( cause instanceof ReflectiveOperationException && cause.getCause() instanceof TezException ) 149 LOG.warn( "unable to construct timeline server client", cause.getCause() ); 150 else if( cause instanceof ReflectiveOperationException && cause.getCause() instanceof NoSuchMethodError ) 151 LOG.warn( "unable to construct timeline server client, check for compatible Tez version, current: {}", getPlatformVersion(), cause.getCause() ); 152 else 153 LOG.warn( "unable to construct timeline server client", exception ); 154 } 155 156 return null; 157 } 158 159 public static String getTrackingURL( TezClient tezClient, DAGClient dagClient ) 160 { 161 if( tezClient == null || dagClient == null ) 162 return null; 163 164 try 165 { 166 ApplicationId applicationId = tezClient.getAppMasterApplicationId(); 167 FrameworkClient frameworkClient = getFrameworkClient( dagClient ); 168 169 if( frameworkClient == null ) 170 { 171 LOG.info( "unable to get framework client" ); 172 return null; 173 } 174 175 ApplicationReport report = frameworkClient.getApplicationReport( applicationId ); 176 177 if( report != null ) 178 return report.getTrackingUrl(); 179 180 } 181 catch( YarnException | IOException exception ) 182 { 183 LOG.info( "unable to get tracking url" ); 184 LOG.debug( "exception retrieving application report", exception ); 185 } 186 187 return null; 188 } 189 190 private static FrameworkClient getFrameworkClient( DAGClient dagClient ) 191 { 192 if( dagClient instanceof TezTimelineClient ) 193 return ( (TezTimelineClient) dagClient ).getFrameworkClient(); 194 195 return Util.returnInstanceFieldIfExistsSafe( dagClient, "frameworkClient" ); 196 } 197 }