【环境】 emqx v3.1.0
【场景】
客户端发送消息时,消息大小超过1MB时,connection有成功有失败的,延迟一段时间后,提示NPE错误
参见如下:
Exception in thread "hawtdispatch-DEFAULT-3" java.lang.NullPointerException
at org.fusesource.mqtt.client.CallbackConnection$7.run(CallbackConnection.java:450)
at org.fusesource.hawtdispatch.transport.HeartBeatMonitor$3.run(HeartBeatMonitor.java:87)
at org.fusesource.hawtdispatch.transport.HeartBeatMonitor$1.run(HeartBeatMonitor.java:65)
at org.fusesource.hawtdispatch.internal.SerialDispatchQueue.run(SerialDispatchQueue.java:100)
at org.fusesource.hawtdispatch.internal.pool.SimpleThread.run(SimpleThread.java:77)
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:35)
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:25)
at org.fusesource.hawtdispatch.util.ThreadLocalPool.checkout(ThreadLocalPool.java:78)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.allocateNextWriteBuffer(AbstractProtocolCodec.java:153)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.write(AbstractProtocolCodec.java:137)
at org.fusesource.hawtdispatch.transport.TcpTransport.offer(TcpTransport.java:653)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:389)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:320)
at org.fusesource.mqtt.client.CallbackConnection$5.onTransportConnected(CallbackConnection.java:296)
at org.fusesource.hawtdispatch.transport.TcpTransport.onConnected(TcpTransport.java:605)
at org.fusesource.hawtdispatch.transport.TcpTransport$2$1$1.run(TcpTransport.java:494)
at org.fusesource.hawtdispatch.internal.NioDispatchSource$3.run(NioDispatchSource.java:209)
at org.fusesource.hawtdispatch.internal.SerialDispatchQueue.run(SerialDispatchQueue.java:100)
at org.fusesource.hawtdispatch.internal.pool.SimpleThread.run(SimpleThread.java:77)
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:35)
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:25)
at org.fusesource.hawtdispatch.util.ThreadLocalPool.checkout(ThreadLocalPool.java:78)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.allocateNextWriteBuffer(AbstractProtocolCodec.java:153)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.write(AbstractProtocolCodec.java:137)
at org.fusesource.hawtdispatch.transport.TcpTransport.offer(TcpTransport.java:653)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:389)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:320)
at org.fusesource.mqtt.client.CallbackConnection$5.onTransportConnected(CallbackConnection.java:296)
at org.fusesource.hawtdispatch.transport.TcpTransport.onConnected(TcpTransport.java:605)
at org.fusesource.hawtdispatch.transport.TcpTransport$2$1$1.run(TcpTransport.java:494)
at org.fusesource.hawtdispatch.internal.NioDispatchSource$3.run(NioDispatchSource.java:209)
at org.fusesource.hawtdispatch.internal.SerialDispatchQueue.run(SerialDispatchQueue.java:100)
at org.fusesource.hawtdispatch.internal.pool.SimpleThread.run(SimpleThread.java:77)
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:35)
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:25)
at org.fusesource.hawtdispatch.util.ThreadLocalPool.checkout(ThreadLocalPool.java:78)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.allocateNextWriteBuffer(AbstractProtocolCodec.java:153)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.write(AbstractProtocolCodec.java:137)
at org.fusesource.hawtdispatch.transport.TcpTransport.offer(TcpTransport.java:653)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:389)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:320)
at org.fusesource.mqtt.client.CallbackConnection$5.onTransportConnected(CallbackConnection.java:296)
at org.fusesource.hawtdispatch.transport.TcpTransport.onConnected(TcpTransport.java:605)
at org.fusesource.hawtdispatch.transport.TcpTransport$2$1$1.run(TcpTransport.java:494)
at org.fusesource.hawtdispatch.internal.NioDispatchSource$3.run(NioDispatchSource.java:209)
at org.fusesource.hawtdispatch.internal.SerialDispatchQueue.run(SerialDispatchQueue.java:100)
at org.fusesource.hawtdispatch.internal.pool.SimpleThread.run(SimpleThread.java:77)
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:35)
at org.fusesource.hawtdispatch.util.BufferPool.create(BufferPool.java:25)
at org.fusesource.hawtdispatch.util.ThreadLocalPool.checkout(ThreadLocalPool.java:78)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.allocateNextWriteBuffer(AbstractProtocolCodec.java:153)
at org.fusesource.hawtdispatch.transport.AbstractProtocolCodec.write(AbstractProtocolCodec.java:137)
at org.fusesource.hawtdispatch.transport.TcpTransport.offer(TcpTransport.java:653)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:389)
at org.fusesource.mqtt.client.CallbackConnection$LoginHandler.onSuccess(CallbackConnection.java:320)
at org.fusesource.mqtt.client.CallbackConnection$5.onTransportConnected(CallbackConnection.java:296)
at org.fusesource.hawtdispatch.transport.TcpTransport.onConnected(TcpTransport.java:605)
at org.fusesource.hawtdispatch.transport.TcpTransport$2$1$1.run(TcpTransport.java:494)
at org.fusesource.hawtdispatch.internal.NioDispatchSource$3.run(NioDispatchSource.java:209)
at org.fusesource.hawtdispatch.internal.SerialDispatchQueue.run(SerialDispatchQueue.java:100)
at org.fusesource.hawtdispatch.internal.pool.SimpleThread.run(SimpleThread.java:77)
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch timer" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-1" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-4" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: Java heap space
Exception in thread "hawtdispatch-DEFAULT-2" java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "hawtdispatch-DEFAULT-2" Exception in thread "hawtdispatch-DEFAULT-2" Exception in thread "hawtdispatch-DEFAULT-2" Exception in thread "hawtdispatch-DEFAULT-3" java.lang.OutOfMemoryError: Java heap space
【分析】 jmap -heap
Heap Usage:
PS Young Generation
Eden Space:
capacity = 234356736 (223.5MB)
used = 234241664 (223.3902587890625MB)
free = 115072 (0.1097412109375MB)
99.95089878705257% used
From Space:
capacity = 234356736 (223.5MB)
used = 27263184 (26.000198364257812MB)
free = 207093552 (197.4998016357422MB)
11.633198373269714% used
To Space:
capacity = 234356736 (223.5MB)
used = 0 (0.0MB)
free = 234356736 (223.5MB)
0.0% used
PS Old Generation
capacity = 1406664704 (1341.5MB)
used = 1406251136 (1341.1055908203125MB)
free = 413568 (0.3944091796875MB)
99.9705993902581% used
3391 interned Strings occupying 287112 bytes.
****************************************
查看subscriber端,没有消息收到,切换消息(256KB)发布是ok的,考虑EMQ X端参数限制问题
****************************************
【解决】
查看EMQ X配置,注意EMQ X按zone区分,内部和外部交互,此场景数据客户端与EMQ X交互,走external通道
因此查看exernal对消息package大小的限制,默认是1MB的,如果要发送更大消息,将此参数【zone.external.max_packet_size】调大即可。
vim
## Maximum MQTT packet size allowed. ## ## Value: Bytes ## Default: 1MB # remedied by can_do on 2019-06-28 zone.external.max_packet_size = 2MB
注意以下参数不用调:
## Maximum MQTT packet size allowed. ## ## Value: Bytes ## Default: 1MB # remedied by can_do on 2019-07-01 mqtt.max_packet_size = 1MB
********************************************