源码目录
初始化 SparkContext 时，会创建 TaskScheduler 和 DAGScheduler。
1 创建TaskScheduler
调用SparkContext.createTaskScheduler(this, master, deployMode)创建TaskScheduler。
- 进入
org.apache.spark.SparkContext.scala
/**
* Create a task scheduler based on a given master URL.
* Return a 2-tuple of the scheduler backend and the task scheduler.
*/
private def createTaskScheduler(
sc: SparkContext,
master: String,
deployMode: String): (SchedulerBackend, TaskScheduler) = {
import SparkMasterRegex._
// When running locally, don't try to re-execute tasks on failure.
val MAX_LOCAL_TASK_FAILURES = 1
master match {
case "local" =>
val scheduler = new TaskSchedulerImpl(sc, MAX_LOCAL_TASK_FAILURES, isLocal = true)
val backend = new LocalSchedulerBackend(sc.getConf, scheduler, 1)
scheduler.initialize(backend)
(backend, scheduler)
case LOCAL_N_REGEX(threads) =>
def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
// local[*] estimates the number of cores on the machine; local[N] uses exactly N threads.
val threadCount = if (threads == "*") localCpuCount else threads.toInt
if (threadCount <= 0) {
throw new SparkException(s"Asked to run locally with $threadCount threads")
}
val scheduler = new TaskSchedulerImpl(sc, MAX_LOCAL_TASK_FAILURES, isLocal = true)
val backend = new LocalSchedulerBackend(sc.getConf, scheduler, threadCount)
scheduler.initialize(backend)
(backend, scheduler)
case LOCAL_N_FAILURES_REGEX(threads, maxFailures) =>
def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
// local[*, M] means the number of cores on the computer with M failures
// local[N, M] means exactly N threads with M failures
val threadCount = if (threads == "*") localCpuCount else threads.toInt
val scheduler = new TaskSchedulerImpl(sc, maxFailures.toInt, isLocal = true)
val backend = new LocalSchedulerBackend(sc.getConf, scheduler, threadCount)
scheduler.initialize(backend)
(backend, scheduler)
case SPARK_REGEX(sparkUrl) =>
val scheduler = new TaskSchedulerImpl(sc)
val masterUrls = sparkUrl.split(",").map("spark://" + _)
val backend = new StandaloneSchedulerBackend(scheduler, sc, masterUrls)
scheduler.initialize(backend)
(backend, scheduler)
case LOCAL_CLUSTER_REGEX(numSlaves, coresPerSlave, memoryPerSlave) =>
// Check to make sure memory requested <= memoryPerSlave. Otherwise Spark will just hang.
val memoryPerSlaveInt = memoryPerSlave.toInt
if (sc.executorMemory > memoryPerSlaveInt) {
throw new SparkException(
"Asked to launch cluster with %d MB RAM / worker but requested %d MB/worker".format(
memoryPerSlaveInt, sc.executorMemory))
}
val scheduler = new TaskSchedulerImpl(sc)
val localCluster = new LocalSparkCluster(
numSlaves.toInt, coresPerSlave.toInt, memoryPerSlaveInt, sc.conf)
val masterUrls = localCluster.start()
val backend = new StandaloneSchedulerBackend(scheduler, sc, masterUrls)
scheduler.initialize(backend)
backend.shutdownCallback = (backend: StandaloneSchedulerBackend) => {
localCluster.stop()
}
(backend, scheduler)
case masterUrl =>
val cm = getClusterManager(masterUrl) match {
case Some(clusterMgr) => clusterMgr
case None => throw new SparkException("Could not parse Master URL: '" + master + "'")
}
try {
val scheduler = cm.createTaskScheduler(sc, masterUrl)
val backend = cm.createSchedulerBackend(sc, masterUrl, scheduler)
cm.initialize(scheduler, backend)
(backend, scheduler)
} catch {
case se: SparkException => throw se
case NonFatal(e) =>
throw new SparkException("External scheduler cannot be instantiated", e)
}
}
}
根据master不同匹配不同的分支。
假设master匹配SPARK_REGEX(spark://(.*))，则进入SPARK_REGEX对应分支。
case SPARK_REGEX(sparkUrl) =>
val scheduler = new TaskSchedulerImpl(sc)
val masterUrls = sparkUrl.split(",").map("spark://" + _)
val backend = new StandaloneSchedulerBackend(scheduler, sc, masterUrls)
scheduler.initialize(backend)
(backend, scheduler)
- 创建TaskSchedulerImpl；
- 切分sparkUrl，分别加上
spark://前缀得到masterUrls； - 创建StandaloneSchedulerBackend；
- 调用scheduler.initialize(backend)初始化TaskScheduler；
- 返回 (backend, scheduler)。
顺便看一下类似于 Spark on YARN 的情况：
case masterUrl =>
val cm = getClusterManager(masterUrl) match {
case Some(clusterMgr) => clusterMgr
case None => throw new SparkException("Could not parse Master URL: '" + master + "'")
}
try {
val scheduler = cm.createTaskScheduler(sc, masterUrl)
val backend = cm.createSchedulerBackend(sc, masterUrl, scheduler)
cm.initialize(scheduler, backend)
(backend, scheduler)
} catch {
case se: SparkException => throw se
case NonFatal(e) =>
throw new SparkException("External scheduler cannot be instantiated", e)
}
- 调用 getClusterManager 创建 ExternalClusterManager；
- 利用 ExternalClusterManager 创建 TaskScheduler 和 SchedulerBackend；
- 调用cm.initialize(scheduler, backend)初始化ExternalClusterManager；
- 返回 (backend, scheduler)。
可以发现两者基本流程类似，区别在于：
前者使用Spark框架内的TaskSchedulerImpl和StandaloneSchedulerBackend创建TaskScheduler 和 SchedulerBackend；
后者使用外部集群管理器 ExternalClusterManager 去创建 TaskScheduler 和 SchedulerBackend。
- 进入
org.apache.spark.scheduler.TaskSchedulerImpl.scala
var backend: SchedulerBackend = null
private var schedulableBuilder: SchedulableBuilder = null
val rootPool: Pool = new Pool("", schedulingMode, 0, 0)
def initialize(backend: SchedulerBackend) {
this.backend = backend
schedulableBuilder = {
schedulingMode match {
case SchedulingMode.FIFO =>
new FIFOSchedulableBuilder(rootPool)
case SchedulingMode.FAIR =>
new FairSchedulableBuilder(rootPool, conf)
case _ =>
throw new IllegalArgumentException(s"Unsupported $SCHEDULER_MODE_PROPERTY: " +
s"$schedulingMode")
}
}
schedulableBuilder.buildPools()
}
- 根据 schedulingMode 去匹配 SchedulingMode，而 schedulingMode 由
spark.scheduler.mode配置，默认FIFO； - 调用 schedulableBuilder.buildPools() 创建调度池。
以默认的FIFO模式为例：
private[spark] class FIFOSchedulableBuilder(val rootPool: Pool)
extends SchedulableBuilder with Logging {
override def buildPools() {
// nothing
}
override def addTaskSetManager(manager: Schedulable, properties: Properties) {
rootPool.addSchedulable(manager)
}
}
2 创建DAGScheduler
调用_dagScheduler = new DAGScheduler(this)创建DAGScheduler。
- 进入
org.apache.spark.scheduler.DAGScheduler.scala
private[spark] class DAGScheduler(
private[scheduler] val sc: SparkContext,
private[scheduler] val taskScheduler: TaskScheduler,
listenerBus: LiveListenerBus,
mapOutputTracker: MapOutputTrackerMaster,
blockManagerMaster: BlockManagerMaster,
env: SparkEnv,
clock: Clock = new SystemClock())
extends Logging {
def this(sc: SparkContext, taskScheduler: TaskScheduler) = {
this(
sc,
taskScheduler,
sc.listenerBus,
sc.env.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster],
sc.env.blockManager.master,
sc.env)
}
def this(sc: SparkContext) = this(sc, sc.taskScheduler)
// 省略部分内容
}
DAGScheduler 构造函数中引入了 TaskScheduler，因为 DAGScheduler 划分完 Stage 后需要调用 TaskScheduler 去提交任务。
3 总结
- 初始化 SparkContext 时会根据 master 不同创建不同的 TaskScheduler 和 SchedulerBackend（例如：spark框架中创建的是 TaskSchedulerImpl 和 StandaloneSchedulerBackend，yarn等框架中是利用 ExternalClusterManager 创建各自的 TaskScheduler 和 SchedulerBackend）；
- 利用 TaskScheduler.initialize(SchedulerBackend) 方法，将 SchedulerBackend 关联到 TaskScheduler 上，同时初始化 TaskScheduler 的调度模式（FIFO / FAIR）；
- 初始化 SparkContext 时创建 DAGScheduler，并将 TaskScheduler 加入到 DAGScheduler 中。