EsRejectedExecutionExceptionindices:data/read/mget[shard]异常处理与线程池设置

报错

• elasticsearch查询失败
• java日志：

org.frameworkset.elasticsearch.ElasticSearchException: java.net.SocketTimeoutException: Read timed out
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient._executeHttp(ElasticSearchRestClient.java:636)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:493)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:661)
	at org.frameworkset.elasticsearch.client.ElasticSearchRestClient.executeHttp(ElasticSearchRestClient.java:467)
	at org.frameworkset.elasticsearch.client.RestClientUtil.getDocument(RestClientUtil.java:982)
	at org.frameworkset.elasticsearch.client.RestClientUtil.getDocument(RestClientUtil.java:1087)
	at cn.lonsun.es.base.BbossUtil.getDocument(BbossUtil.java:225)
	at cn.lonsun.searchEngine.impl.IndexEsServiceImpl.getOneById(IndexEsServiceImpl.java:593)
	at cn.lonsun.searchEngine.impl.SearchEsServiceImpl.scrollSearchForPage(SearchEsServiceImpl.java:283)

• es日志

[2020-06-09T11:06:24,198][DEBUG][o.e.a.g.TransportShardMultiGetAction] [node-1-master-8-9200] null: failed to execute [org.elasticsearch.action.get.MultiGetShardRequest@11e6ef27]
org.elasticsearch.transport.RemoteTransportException: [node-1-master-8-9200][192.168.4.8:9300][indices:data/read/mget[shard][s]]
Caused by: org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected execution of org.elasticsearch.transport.TransportService$7@198d4842 on EsThreadPoolExecutor[get, queue capacity = 1000, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@171dd8cf[Running, pool size = 8, active threads = 8, queued tasks = 1000, completed tasks = 9786245]]
	at org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:50) ~[elasticsearch-5.5.2.jar:5.5.2]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) ~[?:1.8.0_201]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) ~[?:1.8.0_201]
	at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.doExecute(EsThreadPoolExecutor.java:94) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor.execute(EsThreadPoolExecutor.java:89) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendLocalRequest(TransportService.java:640) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.access$000(TransportService.java:74) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService$3.sendRequest(TransportService.java:137) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:586) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:519) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:494) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.perform(TransportSingleShardAction.java:233) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.onFailure(TransportSingleShardAction.java:196) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction.access$1300(TransportSingleShardAction.java:123) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$AsyncSingleAction$2.handleException(TransportSingleShardAction.java:252) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.lambda$handleException$16(TcpTransport.java:1467) ~[elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.common.util.concurrent.EsExecutors$1.execute(EsExecutors.java:110) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.handleException(TcpTransport.java:1465) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.handlerResponseError(TcpTransport.java:1457) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1401) [elasticsearch-5.5.2.jar:5.5.2]
	at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201]

原因

• 高并发请求，线程池不够用，es拒绝服务
• 参考：EsRejectedExecutionException排错与线程池类型
• 官方文档：https://www.elastic.co/guide/en/elasticsearch/reference/5.5/modules-threadpool.html

解决

• 重启。是的，最简单粗暴，最快捷的处理方式，就是重启。如果是集群部署，可以挨个重启，可能在重启到某一个的时候就会发现服务可用了，继续把所有节点重启一遍。
• 更好配置的服务器，更多的集群节点。我们生产用的es服务器，配置是四核八线程32G内存，集群节点也就3个。是比较低的配置，和比较简陋的集群。但数据量较大，搜索并发较高时，es服务撑不住了，导致连接池爆了。
• 目前报错是使用Rest client接口，使用的是HTTP API请求，连接池爆了报错的，后续会改成Transport client连接，是tcp连接，压测下是否有问题。
• 未完待续