scheduler

Type Members

1. case class AccumulableInfo extends Product with Serializable

   

   :: DeveloperApi :: Information about an org.apache.spark.Accumulable modified during a task or stage.

   :: DeveloperApi :: Information about an org.apache.spark.Accumulable modified during a task or stage.

   Annotations

   @DeveloperApi()

   Note

   Once this is JSON serialized the types of update and value will be lost and be cast to strings. This is because the user can define an accumulator of any type and it will be difficult to preserve the type in consumers of the event log. This does not apply to internal accumulators that represent task level metrics.

2. class InputFormatInfo extends Logging

   

   :: DeveloperApi :: Parses and holds information about inputFormat (and files) specified as a parameter.

   :: DeveloperApi :: Parses and holds information about inputFormat (and files) specified as a parameter.

   Annotations

   @DeveloperApi()

3. sealed trait JobResult extends AnyRef

   

   :: DeveloperApi :: A result of a job in the DAGScheduler.

   :: DeveloperApi :: A result of a job in the DAGScheduler.

   Annotations

   @DeveloperApi()

4. abstract class SparkListener extends SparkListenerInterface

   

   :: DeveloperApi :: A default implementation for SparkListenerInterface that has no-op implementations for all callbacks.

   :: DeveloperApi :: A default implementation for SparkListenerInterface that has no-op implementations for all callbacks.

   Note that this is an internal interface which might change in different Spark releases.

   Annotations

   @DeveloperApi()

5. case class SparkListenerApplicationEnd(time: Long) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

6. case class SparkListenerApplicationStart(appName: String, appId: Option[String], time: Long, sparkUser: String, appAttemptId: Option[String], driverLogs: Option[Map[String, String]] = None) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

7. case class SparkListenerBlockManagerAdded(time: Long, blockManagerId: BlockManagerId, maxMem: Long, maxOnHeapMem: Option[Long] = None, maxOffHeapMem: Option[Long] = None) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

8. case class SparkListenerBlockManagerRemoved(time: Long, blockManagerId: BlockManagerId) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

9. case class SparkListenerBlockUpdated(blockUpdatedInfo: BlockUpdatedInfo) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

10. case class SparkListenerEnvironmentUpdate(environmentDetails: Map[String, Seq[(String, String)]]) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

11. trait SparkListenerEvent extends AnyRef

    

    Annotations

    @DeveloperApi() @JsonTypeInfo()

12. case class SparkListenerExecutorAdded(time: Long, executorId: String, executorInfo: ExecutorInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

13. case class SparkListenerExecutorBlacklisted(time: Long, executorId: String, taskFailures: Int) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

14. case class SparkListenerExecutorMetricsUpdate(execId: String, accumUpdates: Seq[(Long, Int, Int, Seq[AccumulableInfo])]) extends SparkListenerEvent with Product with Serializable

    

    Periodic updates from executors.

    Periodic updates from executors.

    execId

    executor id

    accumUpdates

    sequence of (taskId, stageId, stageAttemptId, accumUpdates)

    Annotations

    @DeveloperApi()

15. case class SparkListenerExecutorRemoved(time: Long, executorId: String, reason: String) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

16. case class SparkListenerExecutorUnblacklisted(time: Long, executorId: String) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

17. case class SparkListenerJobEnd(jobId: Int, time: Long, jobResult: JobResult) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

18. case class SparkListenerJobStart(jobId: Int, time: Long, stageInfos: Seq[StageInfo], properties: Properties = null) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

19. case class SparkListenerLogStart(sparkVersion: String) extends SparkListenerEvent with Product with Serializable

    

    An internal class that describes the metadata of an event log.

    An internal class that describes the metadata of an event log.

    Annotations

    @DeveloperApi()

20. case class SparkListenerNodeBlacklisted(time: Long, hostId: String, executorFailures: Int) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

21. case class SparkListenerNodeUnblacklisted(time: Long, hostId: String) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

22. case class SparkListenerSpeculativeTaskSubmitted(stageId: Int) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

23. case class SparkListenerStageCompleted(stageInfo: StageInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

24. case class SparkListenerStageSubmitted(stageInfo: StageInfo, properties: Properties = null) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

25. case class SparkListenerTaskEnd(stageId: Int, stageAttemptId: Int, taskType: String, reason: TaskEndReason, taskInfo: TaskInfo, taskMetrics: TaskMetrics) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

26. case class SparkListenerTaskGettingResult(taskInfo: TaskInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

27. case class SparkListenerTaskStart(stageId: Int, stageAttemptId: Int, taskInfo: TaskInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

28. case class SparkListenerUnpersistRDD(rddId: Int) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

29. class SplitInfo extends AnyRef

    

    Annotations

    @DeveloperApi()

30. class StageInfo extends AnyRef

    

    :: DeveloperApi :: Stores information about a stage to pass from the scheduler to SparkListeners.

    :: DeveloperApi :: Stores information about a stage to pass from the scheduler to SparkListeners.

    Annotations

    @DeveloperApi()

31. class StatsReportListener extends SparkListener with Logging

    

    :: DeveloperApi :: Simple SparkListener that logs a few summary statistics when each stage completes.

    :: DeveloperApi :: Simple SparkListener that logs a few summary statistics when each stage completes.

    Annotations

    @DeveloperApi()

32. class TaskInfo extends AnyRef

    

    :: DeveloperApi :: Information about a running task attempt inside a TaskSet.

    :: DeveloperApi :: Information about a running task attempt inside a TaskSet.

    Annotations

    @DeveloperApi()