org.apache.spark.scheduler
Class StatsReportListener
Object
org.apache.spark.scheduler.StatsReportListener
- All Implemented Interfaces:
- Logging, SparkListener
public class StatsReportListener
- extends Object
- implements SparkListener, Logging
:: DeveloperApi ::
Simple SparkListener that logs a few summary statistics when each stage completes
Method Summary |
static scala.Option<org.apache.spark.util.Distribution> |
extractDoubleDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
|
static scala.Option<org.apache.spark.util.Distribution> |
extractLongDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
|
static long |
hours()
|
static String |
millisToString(long ms)
Reformat a time interval in milliseconds to a prettier format for output |
static long |
minutes()
|
void |
onStageCompleted(SparkListenerStageCompleted stageCompleted)
Called when a stage completes successfully or fails, with information on the completed stage. |
void |
onTaskEnd(SparkListenerTaskEnd taskEnd)
Called when a task ends |
static int[] |
percentiles()
|
static String |
percentilesHeader()
|
static double[] |
probabilities()
|
static long |
seconds()
|
static void |
showBytesDistribution(String heading,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
|
static void |
showBytesDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt)
|
static void |
showBytesDistribution(String heading,
org.apache.spark.util.Distribution dist)
|
static void |
showDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt,
scala.Function1<Object,String> formatNumber)
|
static void |
showDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt,
String format)
|
static void |
showDistribution(String heading,
org.apache.spark.util.Distribution d,
scala.Function1<Object,String> formatNumber)
|
static void |
showDistribution(String heading,
String format,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
|
static void |
showMillisDistribution(String heading,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
|
static void |
showMillisDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt)
|
Methods inherited from class Object |
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Methods inherited from interface org.apache.spark.scheduler.SparkListener |
onApplicationEnd, onApplicationStart, onBlockManagerAdded, onBlockManagerRemoved, onEnvironmentUpdate, onExecutorAdded, onExecutorMetricsUpdate, onExecutorRemoved, onJobEnd, onJobStart, onStageSubmitted, onTaskGettingResult, onTaskStart, onUnpersistRDD |
Methods inherited from interface org.apache.spark.Logging |
initializeIfNecessary, initializeLogging, isTraceEnabled, log_, log, logDebug, logDebug, logError, logError, logInfo, logInfo, logName, logTrace, logTrace, logWarning, logWarning |
StatsReportListener
public StatsReportListener()
percentiles
public static int[] percentiles()
probabilities
public static double[] probabilities()
percentilesHeader
public static String percentilesHeader()
extractDoubleDistribution
public static scala.Option<org.apache.spark.util.Distribution> extractDoubleDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
extractLongDistribution
public static scala.Option<org.apache.spark.util.Distribution> extractLongDistribution(scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric)
showDistribution
public static void showDistribution(String heading,
org.apache.spark.util.Distribution d,
scala.Function1<Object,String> formatNumber)
showDistribution
public static void showDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt,
scala.Function1<Object,String> formatNumber)
showDistribution
public static void showDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt,
String format)
showDistribution
public static void showDistribution(String heading,
String format,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
showBytesDistribution
public static void showBytesDistribution(String heading,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
showBytesDistribution
public static void showBytesDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt)
showBytesDistribution
public static void showBytesDistribution(String heading,
org.apache.spark.util.Distribution dist)
showMillisDistribution
public static void showMillisDistribution(String heading,
scala.Option<org.apache.spark.util.Distribution> dOpt)
showMillisDistribution
public static void showMillisDistribution(String heading,
scala.Function2<TaskInfo,org.apache.spark.executor.TaskMetrics,scala.Option<Object>> getMetric,
scala.collection.Seq<scala.Tuple2<TaskInfo,org.apache.spark.executor.TaskMetrics>> taskInfoMetrics)
seconds
public static long seconds()
minutes
public static long minutes()
hours
public static long hours()
millisToString
public static String millisToString(long ms)
- Reformat a time interval in milliseconds to a prettier format for output
- Parameters:
ms
- (undocumented)
- Returns:
- (undocumented)
onTaskEnd
public void onTaskEnd(SparkListenerTaskEnd taskEnd)
- Description copied from interface:
SparkListener
- Called when a task ends
- Specified by:
onTaskEnd
in interface SparkListener
- Parameters:
taskEnd
- (undocumented)
onStageCompleted
public void onStageCompleted(SparkListenerStageCompleted stageCompleted)
- Description copied from interface:
SparkListener
- Called when a stage completes successfully or fails, with information on the completed stage.
- Specified by:
onStageCompleted
in interface SparkListener
- Parameters:
stageCompleted
- (undocumented)