org.apache.spark.util

Class JsonProtocol

Object

org.apache.spark.util.JsonProtocol

public class JsonProtocol
extends Object

Serializes SparkListener events to/from JSON. This protocol provides strong backwards- and forwards-compatibility guarantees: any version of Spark should be able to read JSON output written by any other version, including newer versions.

JsonProtocolSuite contains backwards-compatibility tests which check that the current version of JsonProtocol is able to read output written by earlier versions. We do not currently have tests for reading newer JSON output with older Spark versions.

To ensure that we provide these guarantees, follow these rules when modifying these methods:

- Never delete any JSON fields. - Any new JSON fields should be optional; use Utils.jsonOption when reading these fields in *FromJson methods.

Constructor Summary

Constructors

Constructor and Description
JsonProtocol()

Method Summary

Methods

Modifier and Type	Method and Description
static AccumulableInfo	accumulableInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	accumulableInfoToJson(AccumulableInfo accumulableInfo)
static SparkListenerApplicationEnd	applicationEndFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	applicationEndToJson(SparkListenerApplicationEnd applicationEnd)
static SparkListenerApplicationStart	applicationStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	applicationStartToJson(SparkListenerApplicationStart applicationStart)
static SparkListenerBlockManagerAdded	blockManagerAddedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerAddedToJson(SparkListenerBlockManagerAdded blockManagerAdded)
static BlockManagerId	blockManagerIdFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerIdToJson(BlockManagerId blockManagerId)
static SparkListenerBlockManagerRemoved	blockManagerRemovedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerRemovedToJson(SparkListenerBlockManagerRemoved blockManagerRemoved)
static BlockStatus	blockStatusFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockStatusToJson(BlockStatus blockStatus)
static SparkListenerEnvironmentUpdate	environmentUpdateFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	environmentUpdateToJson(SparkListenerEnvironmentUpdate environmentUpdate)
static Exception	exceptionFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	exceptionToJson(Exception exception)
static org.apache.spark.executor.InputMetrics	inputMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	inputMetricsToJson(org.apache.spark.executor.InputMetrics inputMetrics)
static SparkListenerJobEnd	jobEndFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobEndToJson(SparkListenerJobEnd jobEnd)
static JobResult	jobResultFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobResultToJson(JobResult jobResult)
static SparkListenerJobStart	jobStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobStartToJson(SparkListenerJobStart jobStart)
static scala.collection.Map<String,String>	mapFromJson(org.json4s.JsonAST.JValue json) -------------------------------- * Util JSON deserialization methods | ---------------------------------
static org.json4s.JsonAST.JValue	mapToJson(scala.collection.Map<String,String> m) ------------------------------ * Util JSON serialization methods | -------------------------------
static org.apache.spark.executor.OutputMetrics	outputMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	outputMetricsToJson(org.apache.spark.executor.OutputMetrics outputMetrics)
static java.util.Properties	propertiesFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	propertiesToJson(java.util.Properties properties)
static RDDInfo	rddInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	rddInfoToJson(RDDInfo rddInfo)
static org.apache.spark.executor.ShuffleReadMetrics	shuffleReadMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	shuffleReadMetricsToJson(org.apache.spark.executor.ShuffleReadMetrics shuffleReadMetrics)
static org.apache.spark.executor.ShuffleWriteMetrics	shuffleWriteMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	shuffleWriteMetricsToJson(org.apache.spark.executor.ShuffleWriteMetrics shuffleWriteMetrics)
static SparkListenerEvent	sparkEventFromJson(org.json4s.JsonAST.JValue json) --------------------------------------------------- * JSON deserialization methods for SparkListenerEvents | ----------------------------------------------------
static org.json4s.JsonAST.JValue	sparkEventToJson(SparkListenerEvent event) ------------------------------------------------- * JSON serialization methods for SparkListenerEvents | --------------------------------------------------
static StackTraceElement[]	stackTraceFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stackTraceToJson(StackTraceElement[] stackTrace)
static SparkListenerStageCompleted	stageCompletedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stageCompletedToJson(SparkListenerStageCompleted stageCompleted)
static StageInfo	stageInfoFromJson(org.json4s.JsonAST.JValue json) --------------------------------------------------------------------- * JSON deserialization methods for classes SparkListenerEvents depend on | ----------------------------------------------------------------------
static org.json4s.JsonAST.JValue	stageInfoToJson(StageInfo stageInfo) ------------------------------------------------------------------- * JSON serialization methods for classes SparkListenerEvents depend on | --------------------------------------------------------------------
static SparkListenerStageSubmitted	stageSubmittedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stageSubmittedToJson(SparkListenerStageSubmitted stageSubmitted)
static StorageLevel	storageLevelFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	storageLevelToJson(StorageLevel storageLevel)
static SparkListenerTaskEnd	taskEndFromJson(org.json4s.JsonAST.JValue json)
static TaskEndReason	taskEndReasonFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskEndReasonToJson(TaskEndReason taskEndReason)
static org.json4s.JsonAST.JValue	taskEndToJson(SparkListenerTaskEnd taskEnd)
static SparkListenerTaskGettingResult	taskGettingResultFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskGettingResultToJson(SparkListenerTaskGettingResult taskGettingResult)
static TaskInfo	taskInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskInfoToJson(TaskInfo taskInfo)
static org.apache.spark.executor.TaskMetrics	taskMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskMetricsToJson(org.apache.spark.executor.TaskMetrics taskMetrics)
static SparkListenerTaskStart	taskStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskStartToJson(SparkListenerTaskStart taskStart)
static SparkListenerUnpersistRDD	unpersistRDDFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	unpersistRDDToJson(SparkListenerUnpersistRDD unpersistRDD)
static java.util.UUID	UUIDFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	UUIDToJson(java.util.UUID id)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

JsonProtocol

public JsonProtocol()

Method Detail

sparkEventToJson

public static org.json4s.JsonAST.JValue sparkEventToJson(SparkListenerEvent event)

------------------------------------------------- * JSON serialization methods for SparkListenerEvents | --------------------------------------------------

stageSubmittedToJson

public static org.json4s.JsonAST.JValue stageSubmittedToJson(SparkListenerStageSubmitted stageSubmitted)

stageCompletedToJson

public static org.json4s.JsonAST.JValue stageCompletedToJson(SparkListenerStageCompleted stageCompleted)

taskStartToJson

public static org.json4s.JsonAST.JValue taskStartToJson(SparkListenerTaskStart taskStart)

taskGettingResultToJson

public static org.json4s.JsonAST.JValue taskGettingResultToJson(SparkListenerTaskGettingResult taskGettingResult)

taskEndToJson

public static org.json4s.JsonAST.JValue taskEndToJson(SparkListenerTaskEnd taskEnd)

jobStartToJson

public static org.json4s.JsonAST.JValue jobStartToJson(SparkListenerJobStart jobStart)

jobEndToJson

public static org.json4s.JsonAST.JValue jobEndToJson(SparkListenerJobEnd jobEnd)

environmentUpdateToJson

public static org.json4s.JsonAST.JValue environmentUpdateToJson(SparkListenerEnvironmentUpdate environmentUpdate)

blockManagerAddedToJson

public static org.json4s.JsonAST.JValue blockManagerAddedToJson(SparkListenerBlockManagerAdded blockManagerAdded)

blockManagerRemovedToJson

public static org.json4s.JsonAST.JValue blockManagerRemovedToJson(SparkListenerBlockManagerRemoved blockManagerRemoved)

unpersistRDDToJson

public static org.json4s.JsonAST.JValue unpersistRDDToJson(SparkListenerUnpersistRDD unpersistRDD)

applicationStartToJson

public static org.json4s.JsonAST.JValue applicationStartToJson(SparkListenerApplicationStart applicationStart)

applicationEndToJson

public static org.json4s.JsonAST.JValue applicationEndToJson(SparkListenerApplicationEnd applicationEnd)

stageInfoToJson

public static org.json4s.JsonAST.JValue stageInfoToJson(StageInfo stageInfo)

------------------------------------------------------------------- * JSON serialization methods for classes SparkListenerEvents depend on | --------------------------------------------------------------------

taskInfoToJson

public static org.json4s.JsonAST.JValue taskInfoToJson(TaskInfo taskInfo)

accumulableInfoToJson

public static org.json4s.JsonAST.JValue accumulableInfoToJson(AccumulableInfo accumulableInfo)

taskMetricsToJson

public static org.json4s.JsonAST.JValue taskMetricsToJson(org.apache.spark.executor.TaskMetrics taskMetrics)

shuffleReadMetricsToJson

public static org.json4s.JsonAST.JValue shuffleReadMetricsToJson(org.apache.spark.executor.ShuffleReadMetrics shuffleReadMetrics)

shuffleWriteMetricsToJson

public static org.json4s.JsonAST.JValue shuffleWriteMetricsToJson(org.apache.spark.executor.ShuffleWriteMetrics shuffleWriteMetrics)

inputMetricsToJson

public static org.json4s.JsonAST.JValue inputMetricsToJson(org.apache.spark.executor.InputMetrics inputMetrics)

outputMetricsToJson

public static org.json4s.JsonAST.JValue outputMetricsToJson(org.apache.spark.executor.OutputMetrics outputMetrics)

taskEndReasonToJson

public static org.json4s.JsonAST.JValue taskEndReasonToJson(TaskEndReason taskEndReason)

blockManagerIdToJson

public static org.json4s.JsonAST.JValue blockManagerIdToJson(BlockManagerId blockManagerId)

jobResultToJson

public static org.json4s.JsonAST.JValue jobResultToJson(JobResult jobResult)

rddInfoToJson

public static org.json4s.JsonAST.JValue rddInfoToJson(RDDInfo rddInfo)

storageLevelToJson

public static org.json4s.JsonAST.JValue storageLevelToJson(StorageLevel storageLevel)

blockStatusToJson

public static org.json4s.JsonAST.JValue blockStatusToJson(BlockStatus blockStatus)

mapToJson

public static org.json4s.JsonAST.JValue mapToJson(scala.collection.Map<String,String> m)

------------------------------ * Util JSON serialization methods | -------------------------------

propertiesToJson

public static org.json4s.JsonAST.JValue propertiesToJson(java.util.Properties properties)

UUIDToJson

public static org.json4s.JsonAST.JValue UUIDToJson(java.util.UUID id)

stackTraceToJson

public static org.json4s.JsonAST.JValue stackTraceToJson(StackTraceElement[] stackTrace)

exceptionToJson

public static org.json4s.JsonAST.JValue exceptionToJson(Exception exception)

sparkEventFromJson

public static SparkListenerEvent sparkEventFromJson(org.json4s.JsonAST.JValue json)

--------------------------------------------------- * JSON deserialization methods for SparkListenerEvents | ----------------------------------------------------

stageSubmittedFromJson

public static SparkListenerStageSubmitted stageSubmittedFromJson(org.json4s.JsonAST.JValue json)

stageCompletedFromJson

public static SparkListenerStageCompleted stageCompletedFromJson(org.json4s.JsonAST.JValue json)

taskStartFromJson

public static SparkListenerTaskStart taskStartFromJson(org.json4s.JsonAST.JValue json)

taskGettingResultFromJson

public static SparkListenerTaskGettingResult taskGettingResultFromJson(org.json4s.JsonAST.JValue json)

taskEndFromJson

public static SparkListenerTaskEnd taskEndFromJson(org.json4s.JsonAST.JValue json)

jobStartFromJson

public static SparkListenerJobStart jobStartFromJson(org.json4s.JsonAST.JValue json)

jobEndFromJson

public static SparkListenerJobEnd jobEndFromJson(org.json4s.JsonAST.JValue json)

environmentUpdateFromJson

public static SparkListenerEnvironmentUpdate environmentUpdateFromJson(org.json4s.JsonAST.JValue json)

blockManagerAddedFromJson

public static SparkListenerBlockManagerAdded blockManagerAddedFromJson(org.json4s.JsonAST.JValue json)

blockManagerRemovedFromJson

public static SparkListenerBlockManagerRemoved blockManagerRemovedFromJson(org.json4s.JsonAST.JValue json)

unpersistRDDFromJson

public static SparkListenerUnpersistRDD unpersistRDDFromJson(org.json4s.JsonAST.JValue json)

applicationStartFromJson

public static SparkListenerApplicationStart applicationStartFromJson(org.json4s.JsonAST.JValue json)

applicationEndFromJson

public static SparkListenerApplicationEnd applicationEndFromJson(org.json4s.JsonAST.JValue json)

stageInfoFromJson

public static StageInfo stageInfoFromJson(org.json4s.JsonAST.JValue json)

--------------------------------------------------------------------- * JSON deserialization methods for classes SparkListenerEvents depend on | ----------------------------------------------------------------------

taskInfoFromJson

public static TaskInfo taskInfoFromJson(org.json4s.JsonAST.JValue json)

accumulableInfoFromJson

public static AccumulableInfo accumulableInfoFromJson(org.json4s.JsonAST.JValue json)

taskMetricsFromJson

public static org.apache.spark.executor.TaskMetrics taskMetricsFromJson(org.json4s.JsonAST.JValue json)

shuffleReadMetricsFromJson

public static org.apache.spark.executor.ShuffleReadMetrics shuffleReadMetricsFromJson(org.json4s.JsonAST.JValue json)

shuffleWriteMetricsFromJson

public static org.apache.spark.executor.ShuffleWriteMetrics shuffleWriteMetricsFromJson(org.json4s.JsonAST.JValue json)

inputMetricsFromJson

public static org.apache.spark.executor.InputMetrics inputMetricsFromJson(org.json4s.JsonAST.JValue json)

outputMetricsFromJson

public static org.apache.spark.executor.OutputMetrics outputMetricsFromJson(org.json4s.JsonAST.JValue json)

taskEndReasonFromJson

public static TaskEndReason taskEndReasonFromJson(org.json4s.JsonAST.JValue json)

blockManagerIdFromJson

public static BlockManagerId blockManagerIdFromJson(org.json4s.JsonAST.JValue json)

jobResultFromJson

public static JobResult jobResultFromJson(org.json4s.JsonAST.JValue json)

rddInfoFromJson

public static RDDInfo rddInfoFromJson(org.json4s.JsonAST.JValue json)

storageLevelFromJson

public static StorageLevel storageLevelFromJson(org.json4s.JsonAST.JValue json)

blockStatusFromJson

public static BlockStatus blockStatusFromJson(org.json4s.JsonAST.JValue json)

mapFromJson

public static scala.collection.Map<String,String> mapFromJson(org.json4s.JsonAST.JValue json)

-------------------------------- * Util JSON deserialization methods | ---------------------------------

propertiesFromJson

public static java.util.Properties propertiesFromJson(org.json4s.JsonAST.JValue json)

UUIDFromJson

public static java.util.UUID UUIDFromJson(org.json4s.JsonAST.JValue json)

stackTraceFromJson

public static StackTraceElement[] stackTraceFromJson(org.json4s.JsonAST.JValue json)

exceptionFromJson

public static Exception exceptionFromJson(org.json4s.JsonAST.JValue json)