Yarn的主要的异步调度器分析
Yarn的主要的异步调度器分析
采用的事件驱动模型进行编程，实现组件之间的松耦合/解耦、实现异步任务、跟踪状态变化。
简化之后的类图
implements
implements
implements
implements
implements
«interface»
Event
+Enum getType()
+long getTimestamp()
«interface»
EventHandler
+void handle(T event)
«interface»
Dispatcher
+EventHandler getEventHandler()
+void register(Class eventType, EventHandler handler)
AsyncDispatcher
- final BlockingQueue eventQueue
- volatile int lastEventQueueSizeLogged
- Thread eventHandlingThread
- final EventHandler handlerInstance
# final Map, EventHandler> eventDispatchers
+EventHandler getEventHandler()
+void register(Class eventType, EventHandler handler)
+AsyncDispatcher(BlockingQueue eventQueue)
#void dispatch(Event event)
+void start()
+void stop()
SimpleEvent1
- long timestamp
- Enum type
+Enum getType()
+long getTimestamp()
SimpleEvent2
- long timestamp
- Enum type
+Enum getType()
+long getTimestamp()
SimpleEventHandler2
+void handle(Event event)
SimpleEventHandler1
+void handle(SimpleEvent1 event)
«enumeration»
EventEnum1
SimpleEvent11
SimpleEvent12
«enumeration»
EventEnum2
SimpleEvent21
SimpleEvent22
在调度器中注册事件与事件处理器的关系
当事件发生时，调度器将事件转发给处理器进行处理。
组成要素
-
事件：代表不同的事件，通过枚举类可以实现对事件分组，每个分组内的事件可以共用一个事件处理器
-
事件处理器：对不同类型的事件，进行相应的逻辑处理
-
事件调度器：实现对事件与事件处理器的关系映射，将事件转发给事件处理器
异步调度器是核心，核心逻辑
- 通过一个阻塞队列存放事件
- 在内部定义了两个事件处理类，通用事件处理类和复合事件处理类，分别满足事件与事件处理器的一对一关系和一对多关系时
- 定义一个线程专门处理转发队列里面的事件。
实验代码
-
异步调度器的生产实现一定更为复杂，需要处理异常，需要更健壮。
-
对于多个不同组的事件，还可以在抽象一层，有不同类型的事件代表不同的多个事件处理器。
具体的可去阅读Yarn的相关源码，RM中的rmDispatcher就是其核心调度器，管理RM层面的所有事件的调度。源码在ResourceManager.setupDispatcher方法中可以找到入口。
yarn主要的调度器在
- RM中的调度器
- NM的调度器
- ContainerManagerImpl的调度器
Event
package com.donny.dispatcher;
/**
* 定义事件的API的接口
*
* @author [email protected]
* @date 2023/9/8
*/
public interface Event<TYPE extends Enum<TYPE>> {
TYPE getType();
long getTimestamp();
String toString();
}
Dispatcher
package com.donny.dispatcher;
/**
* 事件调度接口
* 它根据事件类型将事件分派给已注册的事件处理程序。
*
* @author [email protected]
* @date 2023/9/8
*/
@SuppressWarnings("rawtypes")
public interface Dispatcher {
EventHandler getEventHandler();
/**
* 为事件类型注册其对应的事件处理器
*
* @param eventType
* @param handler
*/
void register(Class<? extends Enum> eventType, EventHandler handler);
}
EventHandler
package com.donny.dispatcher;
/**
* 处理 T 类型事件的接口
* @author [email protected]
* @date 2023/9/8
*/
@SuppressWarnings("rawtypes")
public interface EventHandler<T extends Event> {
void handle(T event);
}
EventEnum1
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/8
*/
public enum EventEnum1 {
SimpleEvent11,
SimpleEvent12;
}
EventEnum2
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/9
*/
public enum EventEnum2 {
SimpleEvent21,
SimpleEvent22;
}
AsyncDispatcher
package com.donny.dispatcher;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
/**
* 异步调度器
*
* @author [email protected]
* @description
* @date 2023/9/8
*/
@SuppressWarnings("rawtypes")
public class AsyncDispatcher implements Dispatcher {
/**
* 事件阻塞队列
*/
private final BlockingQueue<Event> eventQueue;
private volatile int lastEventQueueSizeLogged = 0;
private Thread eventHandlingThread;
/**
* 事件调度器的Map
*/
protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
private final EventHandler handlerInstance = new GenericEventHandler();
public AsyncDispatcher() {
this(new LinkedBlockingQueue<Event>());
}
public AsyncDispatcher(BlockingQueue<Event> eventQueue) {
this.eventQueue = eventQueue;
this.eventDispatchers = new HashMap<>();
}
@Override
public EventHandler getEventHandler() {
return handlerInstance;
}
@SuppressWarnings("unchecked")
@Override
public void register(Class<? extends Enum> eventType, EventHandler handler) {
EventHandler<Event> registeredHandler = (EventHandler<Event>) eventDispatchers.get(eventType);
if (registeredHandler == null) {
eventDispatchers.put(eventType, handler);
} else if (!(registeredHandler instanceof MultiListenerHandler)) {
/* for multiple listeners of an event add the multiple listener handler */
MultiListenerHandler multiHandler = new MultiListenerHandler();
multiHandler.addHandler(registeredHandler);
multiHandler.addHandler(handler);
eventDispatchers.put(eventType, multiHandler);
} else {
/* already a multilistener, just add to it */
MultiListenerHandler multiHandler = (MultiListenerHandler) registeredHandler;
multiHandler.addHandler(handler);
}
}
/**
* 多路复用事件。将事件发送给订阅此事件的处理程序。
* 通俗的来说就是一个事件引起多个处理程序发生处理。
*
* @see com.donny.dispatcher.EventHandler
*/
static class MultiListenerHandler implements EventHandler<Event> {
List<EventHandler<Event>> listofHandlers;
public MultiListenerHandler() {
listofHandlers = new ArrayList<>();
}
@Override
public void handle(Event event) {
for (EventHandler<Event> handler : listofHandlers) {
handler.handle(event);
}
}
void addHandler(EventHandler<Event> handler) {
listofHandlers.add(handler);
}
}
class GenericEventHandler implements EventHandler<Event> {
public void handle(Event event) {
/* all this method does is enqueue all the events onto the queue */
int qSize = eventQueue.size();
if (qSize != 0 && qSize % 1000 == 0
&& lastEventQueueSizeLogged != qSize) {
lastEventQueueSizeLogged = qSize;
System.out.println(" 事件队列的长度" + qSize);
}
int remCapacity = eventQueue.remainingCapacity();
if (remCapacity < 1000) {
System.out.println(" 事件队列的容量不足" + remCapacity);
}
try {
eventQueue.put(event);
} catch (InterruptedException e) {
System.out.println("AsyncDispatcher thread interrupted");
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
@SuppressWarnings("unchecked")
protected void dispatch(Event event) {
Class<? extends Enum> type = event.getType().getDeclaringClass();
try {
EventHandler handler = eventDispatchers.get(type);
if (handler != null) {
handler.handle(event);
} else {
throw new Exception("No handler for registered for " + type);
}
} catch (Throwable t) {
System.out.println("AsyncDispatcher dispatch 异常");
t.printStackTrace();
}
}
public void start() {
eventHandlingThread = new Thread(createThread());
/**
* The thread name for dispatcher.
*/
String dispatcherThreadName = "AsyncDispatcher event handler";
eventHandlingThread.setName(dispatcherThreadName);
eventHandlingThread.start();
}
public void stop() {
eventHandlingThread.interrupt();
System.out.println("时间处理线程中断时,还有时间未处理个数: " + eventQueue.size());
}
Runnable createThread() {
return () -> {
while (!Thread.currentThread().isInterrupted()) {
Event event;
try {
event = eventQueue.take();
} catch (InterruptedException ie) {
System.out.println("AsyncDispatcher thread interrupted");
ie.printStackTrace();
return;
}
dispatch(event);
}
};
}
}
SimpleEvent1
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/8
*/
public class SimpleEvent1<TYPE extends Enum<TYPE>> implements Event<TYPE> {
private final TYPE type;
private final long timestamp;
public SimpleEvent1(TYPE type) {
this.type = type;
timestamp = -1L;
}
public SimpleEvent1(TYPE type, Long timestamp) {
this.type = type;
this.timestamp = timestamp;
}
@Override
public TYPE getType() {
return type;
}
@Override
public long getTimestamp() {
return timestamp;
}
}
SimpleEvent2
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/9
*/
public class SimpleEvent2<TYPE extends Enum<TYPE>> implements Event<TYPE> {
private final TYPE type;
private final long timestamp;
public SimpleEvent2(TYPE type) {
this.type = type;
timestamp = -1L;
}
public SimpleEvent2(TYPE type, Long timestamp) {
this.type = type;
this.timestamp = timestamp;
}
@Override
public TYPE getType() {
return type;
}
@Override
public long getTimestamp() {
return timestamp;
}
}
SimpleEventHandler1
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/8
*/
public class SimpleEventHandler1 implements EventHandler<SimpleEvent1<EventEnum1>> {
@Override
public void handle(SimpleEvent1<EventEnum1> event) {
switch (event.getType()) {
case SimpleEvent11:
// 执行逻辑
case SimpleEvent12:
// 执行逻辑
default:
break;
}
System.out.println("SimpleEventHandler1处理开始");
System.out.println("SimpleEventHandler1处理事件类型" + event.getType());
System.out.println("SimpleEventHandler1处理结束");
System.out.println("");
}
}
SimpleEventHandler2
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/9
*/
public class SimpleEventHandler2 implements EventHandler<Event> {
@Override
public void handle(Event event) {
System.out.println("SimpleEventHandler2处理开始");
System.out.println("SimpleEventHandler2处理事件类型" + event.getType());
System.out.println("SimpleEventHandler2处理结束");
System.out.println("");
}
}
测试主类入口
package com.donny.dispatcher;
/**
* @author [email protected]
* @description
* @date 2023/9/8
*/
public class Test {
public static void main(String[] args) {
AsyncDispatcher dispatcher = new AsyncDispatcher();
dispatcher.register(EventEnum1.class, new SimpleEventHandler1());
dispatcher.register(EventEnum1.class, new SimpleEventHandler2());
dispatcher.register(EventEnum2.class, new SimpleEventHandler2());
dispatcher.start();
SimpleEvent1<EventEnum1> simpleEvent11 = new SimpleEvent1<>(EventEnum1.SimpleEvent11);
SimpleEvent1<EventEnum1> simpleEvent12 = new SimpleEvent1<>(EventEnum1.SimpleEvent12);
SimpleEvent2<EventEnum2> simpleEvent21 = new SimpleEvent2<>(EventEnum2.SimpleEvent21);
SimpleEvent2<EventEnum2> simpleEvent22 = new SimpleEvent2<>(EventEnum2.SimpleEvent22);
dispatcher.getEventHandler().handle(simpleEvent11);
dispatcher.getEventHandler().handle(simpleEvent12);
dispatcher.getEventHandler().handle(simpleEvent21);
dispatcher.getEventHandler().handle(simpleEvent22);
// dispatcher.stop();
}
}
执行结果1
SimpleEventHandler1处理开始
SimpleEventHandler1处理事件类型SimpleEvent11
SimpleEventHandler1处理结束
SimpleEventHandler2处理开始
SimpleEventHandler2处理事件类型SimpleEvent11
SimpleEventHandler2处理结束
SimpleEventHandler1处理开始
SimpleEventHandler1处理事件类型SimpleEvent12
SimpleEventHandler1处理结束
SimpleEventHandler2处理开始
SimpleEventHandler2处理事件类型SimpleEvent12
SimpleEventHandler2处理结束
SimpleEventHandler2处理开始
SimpleEventHandler2处理事件类型SimpleEvent21
SimpleEventHandler2处理结束
SimpleEventHandler2处理开始
SimpleEventHandler2处理事件类型SimpleEvent22
SimpleEventHandler2处理结束
执行结果2
时间处理线程中断时,还有时间未处理个数: 3
SimpleEventHandler1处理开始
SimpleEventHandler1处理事件类型SimpleEvent11
SimpleEventHandler1处理结束
SimpleEventHandler2处理开始
SimpleEventHandler2处理事件类型SimpleEvent11
SimpleEventHandler2处理结束