中间件kingshard入门(四):jmeter压测出现的问题too many open files

2017/11/30 14:40:34 - ERROR - server.go:[604] - [server] “Run” “accept tcp [::]:9696: accept4: too many open files” “” conn_id=0
在压测过程中，一开始没看ks的系统日志,发现，压测总是到了一定程度的时候，上不去，在sql.log中则是这个提示

2017/11/30 15:09:05 - OK - 0.1ms - 10.11.20.31:54136->0.0.0.0:9696:SET character_set_results = NULL
2017/11/30 15:09:05 - OK - 0.1ms - 10.11.20.31:54136->0.0.0.0:9696:SET autocommit=1
2017/11/30 15:09:05 - OK - 0.3ms - 10.11.20.31:54137->10.11.10.215:3306:/* mysql-connector-java-5.1.44 ( Revision: b3cda4f864902ffdde495b9df93937c3e20009be ) */SELECT  @@session.auto_increment_increment AS auto_increment_increment, @@character_set_client AS character_set_client, @@character_set_connection AS character_set_connection, @@character_set_results AS character_set_results, @@character_set_server AS character_set_server, @@collation_server AS collation_server, @@init_connect AS init_connect, @@interactive_timeout AS interactive_timeout, @@license AS license, @@lower_case_table_names AS lower_case_table_names, @@max_allowed_packet AS max_allowed_packet, @@net_buffer_length AS net_buffer_length, @@net_write_timeout AS net_write_timeout, @@query_cache_size AS query_cache_size, @@query_cache_type AS query_cache_type, @@sql_mode AS sql_mode, @@system_time_zone AS system_time_zone, @@time_zone AS time_zone, @@tx_isolation AS tx_isolation, @@wait_timeout AS wait_timeout
2017/11/30 15:09:05 - OK - 0.0ms - 10.11.20.31:54137->0.0.0.0:9696:SET character_set_results = NULL
2017/11/30 15:09:05 - OK - 0.0ms - 10.11.20.31:54137->0.0.0.0:9696:SET autocommit=1
2017/11/30 15:09:05 - OK - 0.3ms - 10.11.20.31:54137->10.11.10.215:3306:SET sql_mode='NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES'

其实最有价值的还是sys.log中的错误提示。明确的告知了我们是文件句柄打开过多导致的，因为系统未经优化，所以出现了这个问题，进行查证。查看当前系统打开的文件句柄数

[root@gzsd-ks01-101110214 ~]# lsof |grep king|wc -l
13391
#其中系统中的
[root@gzsd-ks01-101110214 ~]# lsof |wc -l
15840

同时也发现是大量的

kingshard 8737 8821    root 1014u     IPv6          208661953       0t0        TCP 10.11.10.214:9696->10.11.20.31:54128 (ESTABLISHED)
kingshard 8737 8821    root 1015u     IPv6          208661955       0t0        TCP 10.11.10.214:9696->10.11.20.31:54129 (ESTABLISHED)
kingshard 8737 8821    root 1016u     IPv6          208661957       0t0        TCP 10.11.10.214:9696->10.11.20.31:54130 (ESTABLISHED)
kingshard 8737 8821    root 1017u     IPv6          208661959       0t0        TCP 10.11.10.214:9696->10.11.20.31:54131 (ESTABLISHED)
kingshard 8737 8821    root 1018u     IPv6          208661961       0t0        TCP 10.11.10.214:9696->10.11.20.31:54132 (ESTABLISHED)
kingshard 8737 8821    root 1019u     IPv6          208661963       0t0        TCP 10.11.10.214:9696->10.11.20.31:54133 (ESTABLISHED)
kingshard 8737 8821    root 1020u     IPv6          208661965       0t0        TCP 10.11.10.214:9696->10.11.20.31:54134 (ESTABLISHED)
kingshard 8737 8821    root 1021u     IPv6          208661967       0t0        TCP 10.11.10.214:9696->10.11.20.31:54135 (ESTABLISHED)
kingshard 8737 8821    root 1022u     IPv6          208661969       0t0        TCP 10.11.10.214:9696->10.11.20.31:54136 (kingshard 8737 8821    root 1014u     IPv6          208661953       0t0        TCP 10.11.10.214:9696->10.11.20.31:54128 (ESTABLISHED)
kingshard 8737 8821    root 1015u     IPv6          208661955       0t0        TCP 10.11.10.214:9696->10.11.20.31:54129 (ESTABLISHED)
kingshard 8737 8821    root 1016u     IPv6          208661957       0t0        TCP 10.11.10.214:9696->10.11.20.31:54130 (ESTABLISHED)
kingshard 8737 8821    root 1017u     IPv6          208661959       0t0        TCP 10.11.10.214:9696->10.11.20.31:54131 (ESTABLISHED)
kingshard 8737 8821    root 1018u     IPv6          208661961       0t0        TCP 10.11.10.214:9696->10.11.20.31:54132 (ESTABLISHED)
kingshard 8737 8821    root 1019u     IPv6          208661963       0t0        TCP 10.11.10.214:9696->10.11.20.31:54133 (ESTABLISHED)
kingshard 8737 8821    root 1020u     IPv6          208661965       0t0        TCP 10.11.10.214:9696->10.11.20.31:54134 (ESTABLISHED)
kingshard 8737 8821    root 1021u     IPv6          208661967       0t0        TCP 10.11.10.214:9696->10.11.20.31:54135 (ESTABLISHED)
kingshard 8737 8821    root 1022u     IPv6          208661969       0t0        TCP 10.11.10.214:9696->10.11.20.31:54136 (ESTABLISHED)
)

对于ESTABLISHED这种，同样需要进行内核优化，优化文件数及TCP连接的问题

# vim /etc/security/limits.conf 
*           soft    nofile          65536
*           hard    nofile          65536
*           soft    nproc           2047
*           hard    nproc           16384
*           soft    stack           10240 
*           hard    stack           20480
*           soft    memlock         3145728
*           hard    memlock         3145728
# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
#3865470566.2  4*1024*1024*1024*0.9
kernel.shmmax = 3865470566
#32*1024*1024*1024*0.9/4 
kernel.shmall = 966367641   
net.ipv4.tcp_max_tw_buckets = 6000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096 131072 1048576
net.ipv4.tcp_wmem = 4096 131072 1048576
net.core.wmem_default = 8388608
net.core.wmem_max = 16777216
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_keepalive_time = 30
net.ipv4.ip_local_port_range = 9000 65500
fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
fs.aio-max-nr = 1048576
vm.swappiness = 10

重启即可