spark 版本定制 第5课：基于案例一节课贯通Spark Streaming流计算框架的运行源码7
先贴下案例源码
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Durations, StreamingContext}
/**
* 感谢王家林老师的知识分享
* 王家林老师名片：
* 中国Spark第一人
* 感谢王家林老师的知识分享
* 新浪微博：http://weibo.com/ilovepains
* 微信公众号：DT_Spark
* 博客：http://blog.sina.com.cn/ilovepains
* 手机：18610086859
* QQ：1740415547
* 邮箱：[email protected]
* YY课堂：每天20:00免费现场授课频道68917580
* 王家林：DT大数据梦工厂创始人、Spark亚太研究院院长和首席专家、大数据培训专家、大数据架构师。
*/
object StreamingWordCountSelfScala {
def main(args: Array[String]) {
val sparkConf = new SparkConf().setMaster("spark://master:7077").setAppName("StreamingWordCountSelfScala")
val ssc = new StreamingContext(sparkConf, Durations.seconds(5)) // 每5秒收割一次数据
val lines = ssc.socketTextStream("localhost", 9999) // 监听 本地9999 socket 端口
val words = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _) // flat map 后 reduce
words.print() // 打印结果
ssc.start() // 启动
ssc.awaitTermination()
ssc.stop(true)
}
}上文已经从源码分析到将Receiver作为RDD提交给Spark，高层调度器将处理此JobSubmitted事件，本文将聚焦于Receiver在Spark 集群中执行的运行时。
// DAGScheduler.scala line 1605
private def doOnReceive(event: DAGSchedulerEvent): Unit = event match {
case JobSubmitted(jobId, rdd, func, partitions, callSite, listener, properties) =>
dagScheduler.handleJobSubmitted(jobId, rdd, func, partitions, callSite, listener, properties)
...
// 其他 case class 的处理
}追踪。有Spark Core基础的读者一定知道。Spark的RDD再定义时是从前往后写逻辑的。计算时，是按照血统从后往前推依赖的。并且按照依赖的宽窄划分stage。
而本例中的RDD的数据只是一个很单纯的Receiver，因此没有依赖。只有一个RDD，而且是ParallelCollectionRDD类型的。
// DAGScheduler.scala line 826
private[scheduler] def handleJobSubmitted(jobId: Int,
finalRDD: RDD[_],
func: (TaskContext, Iterator[_]) => _,
partitions: Array[Int],
callSite: CallSite,
listener: JobListener,
properties: Properties) {
var finalStage: ResultStage = null
try {
// New stage creation may throw an exception if, for example, jobs are run on a
// HadoopRDD whose underlying HDFS files have been deleted.
finalStage = newResultStage(finalRDD, func, partitions, jobId, callSite) // 获取最后的stage
} catch {
case e: Exception =>
logWarning("Creating new stage failed due to exception - job: " + jobId, e)
listener.jobFailed(e)
return
}
val job = new ActiveJob(jobId, finalStage, callSite, listener, properties)
clearCacheLocs()
logInfo("Got job %s (%s) with %d output partitions".format(
job.jobId, callSite.shortForm, partitions.length))
logInfo("Final stage: " + finalStage + " (" + finalStage.name + ")")
logInfo("Parents of final stage: " + finalStage.parents)
logInfo("Missing parents: " + getMissingParentStages(finalStage))
val jobSubmissionTime = clock.getTimeMillis()
jobIdToActiveJob(jobId) = job
activeJobs += job
finalStage.setActiveJob(job)
val stageIds = jobIdToStageIds(jobId).toArray
val stageInfos = stageIds.flatMap(id => stageIdToStage.get(id).map(_.latestInfo))
listenerBus.post(
SparkListenerJobStart(job.jobId, jobSubmissionTime, stageInfos, properties))
submitStage(finalStage) // 提交最后一个stage
submitWaitingStages()
}追踪
本例中，只有一个stage，那么这个stage就是ResultStage
// DAGScheduler.scala line 911
/** Submits stage, but first recursively submits any missing parents. */
private def submitStage(stage: Stage) {
val jobId = activeJobForStage(stage)
if (jobId.isDefined) {
logDebug("submitStage(" + stage + ")")
if (!waitingStages(stage) && !runningStages(stage) && !failedStages(stage)) {
val missing = getMissingParentStages(stage).sortBy(_.id) // 没有依赖的stage，因此没missing
logDebug("missing: " + missing)
if (missing.isEmpty) { // 当missing为空时，就是最后一个stage，提交运行。
logInfo("Submitting " + stage + " (" + stage.rdd + "), which has no missing parents") // 这个日志也印证了这点。
submitMissingTasks(stage, jobId.get)
} else { // 顺便提一下，若stage有父依赖，则递归掉submitStage。因为一个JVM只有一个DAGScheduler，因此waitingStages 作为成员变量也没什么问题。
for (parent <- missing) {
submitStage(parent)
}
waitingStages += stage
}
}
} else {
abortStage(stage, "No active job for stage " + stage.id, None)
}
}submitMissingTasks
本例中stage是ResultStage。
计算partitions个数 ; DAGScheduler line 941
计算数据本地性 ; DAGScheduler line 966
stage信息序列化 ; DAGScheduler line 996
将stage 二进制信息广播至Executor ; DAGScheduler line 1006
taskScheduler提交任务 ; DAGScheduler line 1052
// DAGScheduler.scala line 934
/** Called when stage's parents are available and we can now do its task. */
private def submitMissingTasks(stage: Stage, jobId: Int) {
logDebug("submitMissingTasks(" + stage + ")")
// Get our pending tasks and remember them in our pendingTasks entry
stage.pendingPartitions.clear()
// First figure out the indexes of partition ids to compute.
val partitionsToCompute: Seq[Int] = stage.findMissingPartitions() // 此处是1，有兴趣的课跟踪进去
// ... 略过一些步骤
var taskBinary: Broadcast[Array[Byte]] = null
try {
// For ShuffleMapTask, serialize and broadcast (rdd, shuffleDep).
// For ResultTask, serialize and broadcast (rdd, func).
val taskBinaryBytes: Array[Byte] = stage match {
case stage: ShuffleMapStage =>
closureSerializer.serialize((stage.rdd, stage.shuffleDep): AnyRef).array()
case stage: ResultStage =>
closureSerializer.serialize((stage.rdd, stage.func): AnyRef).array()
}
taskBinary = sc.broadcast(taskBinaryBytes) // 此处，将Stage的序列化后的信息，以广播的形式发到各worker上的Executor上了。
} catch {
// In the case of a failure during serialization, abort the stage.
case e: NotSerializableException =>
abortStage(stage, "Task not serializable: " + e.toString, Some(e))
runningStages -= stage
// Abort execution
return
case NonFatal(e) =>
abortStage(stage, s"Task serialization failed: $e\n${e.getStackTraceString}", Some(e))
runningStages -= stage
return
}
val tasks: Seq[Task[_]] = try {
stage match {
case stage: ShuffleMapStage =>
// 不适用本例的场景
case stage: ResultStage =>
val job = stage.activeJob.get
partitionsToCompute.map { id =>
val p: Int = stage.partitions(id)
val part = stage.rdd.partitions(p)
val locs = taskIdToLocations(id)
new ResultTask(stage.id, stage.latestInfo.attemptId,
taskBinary, part, locs, id, stage.internalAccumulators)
}
}
} catch {
case NonFatal(e) =>
abortStage(stage, s"Task creation failed: $e\n${e.getStackTraceString}", Some(e))
runningStages -= stage
return
}
if (tasks.size > 0) {
logInfo("Submitting " + tasks.size + " missing tasks from " + stage + " (" + stage.rdd + ")")
stage.pendingPartitions ++= tasks.map(_.partitionId)
logDebug("New pending partitions: " + stage.pendingPartitions)
taskScheduler.submitTasks(new TaskSet( // 由TaskScheduler提交出去
tasks.toArray, stage.id, stage.latestInfo.attemptId, jobId, properties))
stage.latestInfo.submissionTime = Some(clock.getTimeMillis())
} else {
// Because we posted SparkListenerStageSubmitted earlier, we should mark
// the stage as completed here in case there are no tasks to run
markStageAsFinished(stage, None)
val debugString = stage match {
case stage: ShuffleMapStage =>
// 忽略
case stage : ResultStage =>
s"Stage ${stage} is actually done; (partitions: ${stage.numPartitions})"
}
logDebug(debugString)
}
}任务提交：taskScheduler.submitTasks。TaskScheduler也是在SparkContext的实例化时创建的，具体实现是TaskSchedulerImpl。见SparkContext.scala line 522。
submitTasks中，最后调用的是backend.reviveOffers。在Standalone集群模式下，backend是CoarseGrainedSchedulerBackend
//TaskSchedulerImpl.scala line 160
override def submitTasks(taskSet: TaskSet) {
// ... 省略的操作
backend.reviveOffers()
}reviveOffers()，
// CoarseGrainedSchedulerBackend.scala line 344
override def reviveOffers() {
driverEndpoint.send(ReviveOffers)
}driver处理消息
// CoarseGrainedSchedulerBackend.scala line 122
case ReviveOffers =>
makeOffers()makeOffers
// CoarseGrainedSchedulerBackend.scala line 189
private def makeOffers() {
// Filter out executors under killing
val activeExecutors = executorDataMap.filterKeys(executorIsAlive)
val workOffers = activeExecutors.map { case (id, executorData) =>
new WorkerOffer(id, executorData.executorHost, executorData.freeCores)
}.toSeq
launchTasks(scheduler.resourceOffers(workOffers))
}launchTasks
// CoarseGrainedSchedulerBackend.scala line 224
private def launchTasks(tasks: Seq[Seq[TaskDescription]]) {
for (task <- tasks.flatten) {
val serializedTask = ser.serialize(task)
if (serializedTask.limit >= akkaFrameSize - AkkaUtils.reservedSizeBytes) {
scheduler.taskIdToTaskSetManager.get(task.taskId).foreach { taskSetMgr =>
try {
var msg = "Serialized task %s:%d was %d bytes, which exceeds max allowed: " +
"spark.akka.frameSize (%d bytes) - reserved (%d bytes). Consider increasing " +
"spark.akka.frameSize or using broadcast variables for large values."
msg = msg.format(task.taskId, task.index, serializedTask.limit, akkaFrameSize,
AkkaUtils.reservedSizeBytes)
taskSetMgr.abort(msg)
} catch {
case e: Exception => logError("Exception in error callback", e)
}
}
}
else {
val executorData = executorDataMap(task.executorId)
executorData.freeCores -= scheduler.CPUS_PER_TASK
executorData.executorEndpoint.send(LaunchTask(new SerializableBuffer(serializedTask))) // 给executor发送消息，处理LaunchTask类型的事件
}
}
}至此，上述所有的操作都是在Driver端执行。
下面就刺激了。要转到Worker上的Executor了
王家林老师名片：
中国Spark第一人
新浪微博：http://weibo.com/ilovepains
微信公众号：DT_Spark
博客：http://blog.sina.com.cn/ilovepains
手机：18610086859
QQ：1740415547
YY课堂：每天20:00免费现场授课频道68917580
王家林：DT大数据梦工厂创始人、Spark亚太研究院院长和首席专家、大数据培训专家、大数据架构师。