序
本文主要研究一下elasticsearch的NodesFaultDetection
NodesFaultDetection
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
public class NodesFaultDetection extends AbstractComponent {
public static interface Listener {
void onNodeFailure(DiscoveryNode node, String reason);
}
private final ThreadPool threadPool;
private final TransportService transportService;
private final boolean connectOnNetworkDisconnect;
private final TimeValue pingInterval;
private final TimeValue pingRetryTimeout;
private final int pingRetryCount;
// used mainly for testing, should always be true
private final boolean registerConnectionListener;
private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList();
private final ConcurrentMap nodesFD = newConcurrentMap();
private final FDConnectionListener connectionListener;
private volatile DiscoveryNodes latestNodes = EMPTY_NODES;
private volatile boolean running = false;
public NodesFaultDetection(Settings settings, ThreadPool threadPool, TransportService transportService) {
super(settings);
this.threadPool = threadPool;
this.transportService = transportService;
this.connectOnNetworkDisconnect = componentSettings.getAsBoolean("connect_on_network_disconnect", true);
this.pingInterval = componentSettings.getAsTime("ping_interval", timeValueSeconds(1));
this.pingRetryTimeout = componentSettings.getAsTime("ping_timeout", timeValueSeconds(30));
this.pingRetryCount = componentSettings.getAsInt("ping_retries", 3);
this.registerConnectionListener = componentSettings.getAsBoolean("register_connection_listener", true);
logger.debug("[node ] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
transportService.registerHandler(PingRequestHandler.ACTION, new PingRequestHandler());
this.connectionListener = new FDConnectionListener();
if (registerConnectionListener) {
transportService.addConnectionListener(connectionListener);
}
}
public NodesFaultDetection start() {
if (running) {
return this;
}
running = true;
return this;
}
public NodesFaultDetection stop() {
if (!running) {
return this;
}
running = false;
return this;
}
public void close() {
stop();
transportService.removeHandler(PingRequestHandler.ACTION);
transportService.removeConnectionListener(connectionListener);
}
//......
} - NodesFaultDetection继承了AbstractComponent，它定义了一个CopyOnWriteArrayList类型的listeners，一个ConcurrentMap的nodesFD，connectionListener、latestNodes、running等属性
- 其构造器读取connect_on_network_disconnect(
默认true)、ping_interval(默认1s)、ping_timeout(默认30s)、ping_retries(默认为3)、register_connection_listener(默认true)配置，然后给transportService注册了PingRequestHandler.ACTION的PingRequestHandler，添加了FDConnectionListener - start方法用于设置running为true；stop用于设置running为false；close方法先执行stop，然后从transportService移除PingRequestHandler.ACTION的handler，并移除connectionListener
PingRequestHandler
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
class PingRequestHandler extends BaseTransportRequestHandler {
public static final String ACTION = "discovery/zen/fd/ping";
@Override
public PingRequest newInstance() {
return new PingRequest();
}
@Override
public void messageReceived(PingRequest request, TransportChannel channel) throws Exception {
// if we are not the node we are supposed to be pinged, send an exception
// this can happen when a kill -9 is sent, and another node is started using the same port
if (!latestNodes.localNodeId().equals(request.nodeId)) {
throw new ElasticSearchIllegalStateException("Got pinged as node [" + request.nodeId + "], but I am node [" + latestNodes.localNodeId() + "]");
}
channel.sendResponse(new PingResponse());
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
}
static class PingRequest extends TransportRequest {
// the (assumed) node id we are pinging
private String nodeId;
PingRequest() {
}
PingRequest(String nodeId) {
this.nodeId = nodeId;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
nodeId = in.readString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(nodeId);
}
}
private static class PingResponse extends TransportResponse {
private PingResponse() {
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
}
} - PingRequestHandler的newInstance方法用于创建PingRequest，该对象定义了nodeId属性用于标识它要请求的目标nodeId；而messageReceived方法用于响应PingRequest请求，它会先判断目标nodeId是否跟localNodeId一致，一致的话则返回PingResponse
FDConnectionListener
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
private class FDConnectionListener implements TransportConnectionListener {
@Override
public void onNodeConnected(DiscoveryNode node) {
}
@Override
public void onNodeDisconnected(DiscoveryNode node) {
handleTransportDisconnect(node);
}
}
private void handleTransportDisconnect(DiscoveryNode node) {
if (!latestNodes.nodeExists(node.id())) {
return;
}
NodeFD nodeFD = nodesFD.remove(node);
if (nodeFD == null) {
return;
}
if (!running) {
return;
}
nodeFD.running = false;
if (connectOnNetworkDisconnect) {
try {
transportService.connectToNode(node);
nodesFD.put(node, new NodeFD());
threadPool.schedule(pingInterval, ThreadPool.Names.SAME, new SendPingRequest(node));
} catch (Exception e) {
logger.trace("[node ] [{}] transport disconnected (with verified connect)", node);
notifyNodeFailure(node, "transport disconnected (with verified connect)");
}
} else {
logger.trace("[node ] [{}] transport disconnected", node);
notifyNodeFailure(node, "transport disconnected");
}
}
private void notifyNodeFailure(final DiscoveryNode node, final String reason) {
threadPool.generic().execute(new Runnable() {
@Override
public void run() {
for (Listener listener : listeners) {
listener.onNodeFailure(node, reason);
}
}
});
}- FDConnectionListener在onNodeDisconnected的时候会执行handleTransportDisconnect；该方法会将该node从nodesFD中移除，标记该nodeFD的running为false
- 如果connectOnNetworkDisconnect为true则对该node进行connect，成功则放入nodesFD，并注册对该node进行SendPingRequest的延时任务，延时pingInterval执行；如果connect异常或者connectOnNetworkDisconnect为false，否执行notifyNodeFailure方法
- notifyNodeFailure方法则会触发NodesFaultDetection.Listener.onNodeFailure回调，这里回调ZenDiscovery的NodeFailureListener的onNodeFailure方法
ZenDiscovery
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, DiscoveryNodesProvider {
//......
@Inject
public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool,
TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService,
DiscoveryNodeService discoveryNodeService, ZenPingService pingService) {
super(settings);
this.clusterName = clusterName;
this.threadPool = threadPool;
this.clusterService = clusterService;
this.transportService = transportService;
this.discoveryNodeService = discoveryNodeService;
this.pingService = pingService;
// also support direct discovery.zen settings, for cases when it gets extended
this.pingTimeout = settings.getAsTime("discovery.zen.ping.timeout", settings.getAsTime("discovery.zen.ping_timeout", componentSettings.getAsTime("ping_timeout", componentSettings.getAsTime("initial_ping_timeout", timeValueSeconds(3)))));
this.sendLeaveRequest = componentSettings.getAsBoolean("send_leave_request", true);
this.masterElectionFilterClientNodes = settings.getAsBoolean("discovery.zen.master_election.filter_client", true);
this.masterElectionFilterDataNodes = settings.getAsBoolean("discovery.zen.master_election.filter_data", false);
logger.debug("using ping.timeout [{}], master_election.filter_client [{}], master_election.filter_data [{}]", pingTimeout, masterElectionFilterClientNodes, masterElectionFilterDataNodes);
this.electMaster = new ElectMasterService(settings);
nodeSettingsService.addListener(new ApplySettings());
this.masterFD = new MasterFaultDetection(settings, threadPool, transportService, this);
this.masterFD.addListener(new MasterNodeFailureListener());
this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService);
this.nodesFD.addListener(new NodeFailureListener());
this.publishClusterState = new PublishClusterStateAction(settings, transportService, this, new NewClusterStateListener());
this.pingService.setNodesProvider(this);
this.membership = new MembershipAction(settings, transportService, this, new MembershipListener());
transportService.registerHandler(RejoinClusterRequestHandler.ACTION, new RejoinClusterRequestHandler());
}
protected void doStart() throws ElasticSearchException {
Map nodeAttributes = discoveryNodeService.buildAttributes();
// note, we rely on the fact that its a new id each time we start, see FD and "kill -9" handling
String nodeId = UUID.randomBase64UUID();
localNode = new DiscoveryNode(settings.get("name"), nodeId, transportService.boundAddress().publishAddress(), nodeAttributes);
latestDiscoNodes = new DiscoveryNodes.Builder().put(localNode).localNodeId(localNode.id()).build();
nodesFD.updateNodes(latestDiscoNodes);
pingService.start();
// do the join on a different thread, the DiscoveryService waits for 30s anyhow till it is discovered
asyncJoinCluster();
}
public void publish(ClusterState clusterState) {
if (!master) {
throw new ElasticSearchIllegalStateException("Shouldn't publish state when not master");
}
latestDiscoNodes = clusterState.nodes();
nodesFD.updateNodes(clusterState.nodes());
publishClusterState.publish(clusterState);
}
private class NodeFailureListener implements NodesFaultDetection.Listener {
@Override
public void onNodeFailure(DiscoveryNode node, String reason) {
handleNodeFailure(node, reason);
}
}
private void handleNodeFailure(final DiscoveryNode node, String reason) {
if (lifecycleState() != Lifecycle.State.STARTED) {
// not started, ignore a node failure
return;
}
if (!master) {
// nothing to do here...
return;
}
clusterService.submitStateUpdateTask("zen-disco-node_failed(" + node + "), reason " + reason, new ProcessedClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
DiscoveryNodes.Builder builder = new DiscoveryNodes.Builder()
.putAll(currentState.nodes())
.remove(node.id());
latestDiscoNodes = builder.build();
currentState = newClusterStateBuilder().state(currentState).nodes(latestDiscoNodes).build();
// check if we have enough master nodes, if not, we need to move into joining the cluster again
if (!electMaster.hasEnoughMasterNodes(currentState.nodes())) {
return rejoin(currentState, "not enough master nodes");
}
// eagerly run reroute to remove dead nodes from routing table
RoutingAllocation.Result routingResult = allocationService.reroute(newClusterStateBuilder().state(currentState).build());
return newClusterStateBuilder().state(currentState).routingResult(routingResult).build();
}
@Override
public void clusterStateProcessed(ClusterState clusterState) {
sendInitialStateEventIfNeeded();
}
});
}
//......
} - ZenDiscovery的构造器创建了NodesFaultDetection，并给它添加了NodeFailureListener；该listener实现了NodesFaultDetection.Listener接口，其onNodeFailure回调执行的是handleNodeFailure方法，它会执行ProcessedClusterStateUpdateTask，将该node从currentState.nodes()中移除，然后判断masterNode数量是否满足minimumMasterNodes，不够的话会执行rejoin方法，够的话则执行allocationService.reroute
- 其doStart方法会根据配置文件的node配置创建localNode，然后加入到latestDiscoNodes中，之后执行nodesFD.updateNodes(latestDiscoNodes)方法，然后执行pingService.start()及asyncJoinCluster()
- 其publish方法则根据clusterState的nodes来更新本地的latestDiscoNodes，然后执行nodesFD.updateNodes(latestDiscoNodes)方法，最后执行publishClusterState.publish(clusterState)
NodesFaultDetection.updateNodes
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
public class NodesFaultDetection extends AbstractComponent {
//......
public void updateNodes(DiscoveryNodes nodes) {
DiscoveryNodes prevNodes = latestNodes;
this.latestNodes = nodes;
if (!running) {
return;
}
DiscoveryNodes.Delta delta = nodes.delta(prevNodes);
for (DiscoveryNode newNode : delta.addedNodes()) {
if (newNode.id().equals(nodes.localNodeId())) {
// no need to monitor the local node
continue;
}
if (!nodesFD.containsKey(newNode)) {
nodesFD.put(newNode, new NodeFD());
threadPool.schedule(pingInterval, ThreadPool.Names.SAME, new SendPingRequest(newNode));
}
}
for (DiscoveryNode removedNode : delta.removedNodes()) {
nodesFD.remove(removedNode);
}
}
//......
}- NodesFaultDetection提供了updateNodes方法用于更新自身的latestNodes，该方法调用了nodes.delta(prevNodes)来计算DiscoveryNodes.Delta，它的addedNodes方法返回新增的node，而emovedNodes()方法返回删除的node
- 对于newNode先判断是否在nodesFD，如果不在的话，则会添加到nodesFD中，并注册一个SendPingRequest的延时任务，延时pingInterval执行
- 对于removedNode则将其从nodesFD中移除；handleTransportDisconnect方法也会将一个disconnect的node从ndoesFD中移除，如果重试一次成功则会再次放入nodesFD中
SendPingRequest
elasticsearch-0.90.0/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
private class SendPingRequest implements Runnable {
private final DiscoveryNode node;
private SendPingRequest(DiscoveryNode node) {
this.node = node;
}
@Override
public void run() {
if (!running) {
return;
}
transportService.sendRequest(node, PingRequestHandler.ACTION, new PingRequest(node.id()), options().withHighType().withTimeout(pingRetryTimeout),
new BaseTransportResponseHandler() {
@Override
public PingResponse newInstance() {
return new PingResponse();
}
@Override
public void handleResponse(PingResponse response) {
if (!running) {
return;
}
NodeFD nodeFD = nodesFD.get(node);
if (nodeFD != null) {
if (!nodeFD.running) {
return;
}
nodeFD.retryCount = 0;
threadPool.schedule(pingInterval, ThreadPool.Names.SAME, SendPingRequest.this);
}
}
@Override
public void handleException(TransportException exp) {
// check if the master node did not get switched on us...
if (!running) {
return;
}
if (exp instanceof ConnectTransportException) {
// ignore this one, we already handle it by registering a connection listener
return;
}
NodeFD nodeFD = nodesFD.get(node);
if (nodeFD != null) {
if (!nodeFD.running) {
return;
}
int retryCount = ++nodeFD.retryCount;
logger.trace("[node ] failed to ping [{}], retry [{}] out of [{}]", exp, node, retryCount, pingRetryCount);
if (retryCount >= pingRetryCount) {
logger.debug("[node ] failed to ping [{}], tried [{}] times, each with maximum [{}] timeout", node, pingRetryCount, pingRetryTimeout);
// not good, failure
if (nodesFD.remove(node) != null) {
notifyNodeFailure(node, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum [" + pingRetryTimeout + "] timeout");
}
} else {
// resend the request, not reschedule, rely on send timeout
transportService.sendRequest(node, PingRequestHandler.ACTION, new PingRequest(node.id()),
options().withHighType().withTimeout(pingRetryTimeout), this);
}
}
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
});
}
} - SendPingRequest方法会往目标node发送PingRequest，其超时时间为pingRetryTimeout；其handleResponse方法会判断该node是否在nodesFD中，如果已经被移除了则忽略，如果改nodeFD的running为false，也忽略，否则重置其retryCount，并重新注册SendPingRequest的延时任务，延时pingInterval执行
- 如果请求出现TransportException则判断是否是ConnectTransportException，如果是则忽略，因为该异常已经由往transportService注册的FDConnectionListener的onNodeDisconnected来处理
- 如果是其他异常则增加nodeFD.retryCount，当retryCount大于等于配置的pingRetryCount时，则会将该node从nodesFD中移除，并回调notifyNodeFailure方法，具体就是回调了ZenDiscovery的handleNodeFailure方法；如果没有超过配置的pingRetryCount则会进行重试，重新发送PingRequest请求
小结
- NodesFaultDetection给transportService注册了PingRequestHandler.ACTION的PingRequestHandler，添加了FDConnectionListener；PingRequestHandler用于响应PingRequest请求，返回PingResponse；FDConnectionListener则用于处理ConnectTransportException异常
- FDConnectionListener的onNodeDisconnected方法会将该node从nodesFD中移除，标记该nodeFD的running为false；如果connectOnNetworkDisconnect为true则会重试一次(
对该node进行connect，成功则放入nodesFD，并注册对该node进行SendPingRequest的延时任务，延时pingInterval执行)；如果connect异常或者connectOnNetworkDisconnect为false，否执行notifyNodeFailure方法；notifyNodeFailure方法则会触发NodesFaultDetection.Listener.onNodeFailure回调，这里回调ZenDiscovery的NodeFailureListener的onNodeFailure方法 - ZenDiscovery的doStart方法及publish方法都会执行NodesFaultDetection的updateNodes方法来更新latestNodes，对于新的node则注册延时任务SendPingRequest
- SendPingRequest执行成功时会重置retryCount并继续注册SendPingRequest的延时任务，如果是非TransportException则进行重试，重试次数超过限制则触发notifyNodeFailure，回调NodesFaultDetection.Listener.onNodeFailure方法，这里回调ZenDiscovery的NodeFailureListener的onNodeFailure方法
- ZenDiscovery的NodeFailureListener实现了NodesFaultDetection.Listener接口，其onNodeFailure回调执行的是handleNodeFailure方法，它会执行ProcessedClusterStateUpdateTask，将该node从currentState.nodes()中移除，然后判断masterNode数量是否满足minimumMasterNodes，不够的话会执行rejoin方法，够的话则执行allocationService.reroute
doc
- elasticsearch reference 0.90