【原创】记录几个最近遇到的未解问题
问题一：ejabberd 持续 crashdump
[root@upucore_105 logs]# ls *.dump
erl_crash_20160420-023053.dump erl_crash_20160420-142431.dump erl_crash_20160421-012212.dump erl_crash_20160421-051153.dump erl_crash_20160421-070845.dump erl_crash_20160421-122015.dump
erl_crash_20160420-024120.dump erl_crash_20160420-142731.dump erl_crash_20160421-012310.dump erl_crash_20160421-052453.dump erl_crash_20160421-071828.dump erl_crash_20160421-123153.dump
erl_crash_20160420-024331.dump erl_crash_20160420-143215.dump erl_crash_20160421-013453.dump erl_crash_20160421-052753.dump erl_crash_20160421-074435.dump erl_crash_20160421-124029.dump
erl_crash_20160420-024823.dump erl_crash_20160420-143324.dump erl_crash_20160421-013552.dump erl_crash_20160421-053627.dump erl_crash_20160421-081136.dump erl_crash_20160421-131853.dump
erl_crash_20160420-032253.dump erl_crash_20160420-150153.dump erl_crash_20160421-014439.dump erl_crash_20160421-054022.dump erl_crash_20160421-084953.dump erl_crash_20160421-132339.dump
erl_crash_20160420-034503.dump erl_crash_20160420-160153.dump erl_crash_20160421-014537.dump erl_crash_20160421-054153.dump erl_crash_20160421-085503.dump erl_crash_20160421-134852.dump
erl_crash_20160420-040853.dump erl_crash_20160420-161253.dump erl_crash_20160421-014636.dump erl_crash_20160421-055355.dump erl_crash_20160421-085701.dump erl_crash_20160421-140313.dump
erl_crash_20160420-041253.dump erl_crash_20160420-163453.dump erl_crash_20160421-014953.dump erl_crash_20160421-060534.dump erl_crash_20160421-085953.dump erl_crash_20160421-141056.dump
erl_crash_20160420-050153.dump erl_crash_20160420-174102.dump erl_crash_20160421-020359.dump erl_crash_20160421-060753.dump erl_crash_20160421-090453.dump erl_crash_20160421-141453.dump
erl_crash_20160420-074907.dump erl_crash_20160420-180053.dump erl_crash_20160421-021247.dump erl_crash_20160421-061418.dump erl_crash_20160421-090838.dump erl_crash_20160421-151653.dump
erl_crash_20160420-080959.dump erl_crash_20160420-184253.dump erl_crash_20160421-022715.dump erl_crash_20160421-061714.dump erl_crash_20160421-092406.dump erl_crash_20160421-152315.dump
erl_crash_20160420-085042.dump erl_crash_20160420-191953.dump erl_crash_20160421-023503.dump erl_crash_20160421-062011.dump erl_crash_20160421-092756.dump erl_crash_20160421-153753.dump
erl_crash_20160420-091353.dump erl_crash_20160420-194153.dump erl_crash_20160421-024154.dump erl_crash_20160421-062307.dump erl_crash_20160421-092953.dump erl_crash_20160421-154453.dump
erl_crash_20160420-093153.dump erl_crash_20160420-223553.dump erl_crash_20160421-025234.dump erl_crash_20160421-062953.dump erl_crash_20160421-094516.dump erl_crash_20160421-160301.dump
erl_crash_20160420-102753.dump erl_crash_20160420-231953.dump erl_crash_20160421-025332.dump erl_crash_20160421-063853.dump erl_crash_20160421-102153.dump erl_crash_20160421-160653.dump
erl_crash_20160420-103453.dump erl_crash_20160421-003149.dump erl_crash_20160421-031945.dump erl_crash_20160421-064523.dump erl_crash_20160421-103326.dump erl_crash_20160421-171823.dump
erl_crash_20160420-104653.dump erl_crash_20160421-003253.dump erl_crash_20160421-033617.dump erl_crash_20160421-064820.dump erl_crash_20160421-111837.dump erl_crash_20160421-173053.dump
erl_crash_20160420-112753.dump erl_crash_20160421-004228.dump erl_crash_20160421-041015.dump erl_crash_20160421-065116.dump erl_crash_20160421-112953.dump erl_crash_20160421-173953.dump
erl_crash_20160420-115008.dump erl_crash_20160421-005857.dump erl_crash_20160421-042247.dump erl_crash_20160421-065803.dump erl_crash_20160421-115902.dump erl_crash_20160421-180145.dump
erl_crash_20160420-134303.dump erl_crash_20160421-005956.dump erl_crash_20160421-042642.dump erl_crash_20160421-070059.dump erl_crash_20160421-120853.dump erl_crash_20160421-181715.dump
erl_crash_20160420-140853.dump erl_crash_20160421-010153.dump erl_crash_20160421-044117.dump erl_crash_20160421-070356.dump erl_crash_20160421-121233.dump
[root@upucore_105 logs]#
[root@upucore_105 logs]# ls *.dump|wc -l
125
[root@upucore_105 logs]#
[root@upucore_105 logs]# date
Thu Apr 21 18:21:54 CST 2016
[root@upucore_105 logs]#
[root@upucore_105 logs]# for I in *.dump; do grep "Slogan" $I; echo "----"; done
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
Slogan: Kernel pid terminated (application_controller) ({application_start_failure,kernel,{shutdown,{kernel,start,[normal,[]]}}})
----
[root@upucore_105 logs]#
[root@upucore_105 logs]# ps -A -o args,stime,etime |grep ejabberd
/usr/local/mo_ejabberd/bin/ Apr20 1-15:42:20
...
除了上述错误信息外，之前还看到下面这种
Slogan: init terminating in do_boot ()
问题二：redis 服务被不断 shutdown
_._
_.-``__ ''-._
_.-`` `. `_. ''-._ Redis 2.8.18 (00000000/0) 64 bit
.-`` .-```. ```\/ _.,_ ''-._
( ' , .-` | `, ) Running in stand alone mode
|`-._`-...-` __...-.``-._|'` _.-'| Port: 6379
| `-._ `._ / _.-' | PID: 204410
`-._ `-._ `-./ _.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' | http://redis.io
`-._ `-._`-.__.-'_.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' |
`-._ `-._`-.__.-'_.-' _.-'
`-._ `-.__.-' _.-'
`-._ _.-'
`-.__.-'
[204410] 19 Apr 18:27:35.131 # Server started, Redis version 2.8.18
[204410] 19 Apr 18:27:35.132 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
[204410] 19 Apr 18:27:35.132 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.
[204410] 19 Apr 18:27:35.132 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
[204410] 19 Apr 18:27:35.161 - Accepted 127.0.0.1:16364
[204410] 19 Apr 18:27:35.166 * DB loaded from disk: 0.034 seconds
[204410] 19 Apr 18:27:35.166 * The server is now ready to accept connections on port 6379
[204410] 19 Apr 18:27:35.166 - Client closed connection
[204410] 19 Apr 18:27:35.166 - DB 0: 13573 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:27:35.167 - 0 clients connected (0 slaves), 4797704 bytes in use
[204410] 19 Apr 18:27:35.223 - Accepted 172.16.186.205:17311
[204410] 19 Apr 18:27:36.078 - Accepted 172.16.186.203:16992
[204410] 19 Apr 18:27:36.078 * Slave 172.16.186.203:6379 asks for synchronization
[204410] 19 Apr 18:27:36.079 * Partial resynchronization not accepted: Runid mismatch (Client asked for 'f88ffa4c476425b22a0c1b56932937669b795c0f', I'm '0c4731011b0b911b000c1d70fdc3f907f76ce180')
[204410] 19 Apr 18:27:36.079 * Starting BGSAVE for SYNC with target: disk
[204410] 19 Apr 18:27:36.080 * Background saving started by pid 204415
[204415] 19 Apr 18:27:36.137 * DB saved on disk
[204415] 19 Apr 18:27:36.137 * RDB: 10 MB of memory used by copy-on-write
[204410] 19 Apr 18:27:36.168 * Background saving terminated with success
[204410] 19 Apr 18:27:36.190 * Synchronization with slave 172.16.186.203:6379 succeeded
[204410] 19 Apr 18:27:38.167 - Accepted 127.0.0.1:16418
[204410] 19 Apr 18:27:38.168 - Client closed connection
[204410] 19 Apr 18:27:38.172 - Accepted 172.16.186.205:10391
[204410] 19 Apr 18:27:38.172 - Client closed connection
[204410] 19 Apr 18:27:40.173 - DB 0: 13573 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:27:40.173 - 1 clients connected (1 slaves), 5924936 bytes in use
[204410] 19 Apr 18:27:45.180 - DB 0: 13573 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:27:45.180 - 1 clients connected (1 slaves), 5924648 bytes in use
[204410] 19 Apr 18:27:50.190 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:27:50.191 - 1 clients connected (1 slaves), 5925024 bytes in use
[204410] 19 Apr 18:27:55.199 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:27:55.199 - 1 clients connected (1 slaves), 5926848 bytes in use
[204410] 19 Apr 18:27:58.194 - Client closed connection
[204410] 19 Apr 18:27:58.194 # Connection with slave 172.16.186.203:6379 lost.
[204410] 19 Apr 18:28:00.208 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:00.208 - 1 clients connected (0 slaves), 5870736 bytes in use
[204410] 19 Apr 18:28:02.248 - Accepted 172.16.186.203:17514
[204410] 19 Apr 18:28:02.248 * Slave 172.16.186.203:6379 asks for synchronization
[204410] 19 Apr 18:28:02.248 * Full resync requested by slave 172.16.186.203:6379
[204410] 19 Apr 18:28:02.248 * Starting BGSAVE for SYNC with target: disk
[204410] 19 Apr 18:28:02.250 * Background saving started by pid 205002
[205002] 19 Apr 18:28:02.307 * DB saved on disk
[205002] 19 Apr 18:28:02.308 * RDB: 12 MB of memory used by copy-on-write
[204410] 19 Apr 18:28:02.311 * Background saving terminated with success
[204410] 19 Apr 18:28:02.332 * Synchronization with slave 172.16.186.203:6379 succeeded
[204410] 19 Apr 18:28:05.216 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:05.216 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:10.225 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:10.225 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:15.233 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:15.233 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:20.239 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:20.239 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:25.246 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:25.246 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:30.254 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[204410] 19 Apr 18:28:30.254 - 1 clients connected (1 slaves), 5891664 bytes in use
[204410] 19 Apr 18:28:33.507 - Accepted 127.0.0.1:17448
[204410] 19 Apr 18:28:33.507 # User requested shutdown...
[204410] 19 Apr 18:28:33.508 * Saving the final RDB snapshot before exiting.
[204410] 19 Apr 18:28:33.568 * DB saved on disk
[204410] 19 Apr 18:28:33.568 * Removing the pid file.
[204410] 19 Apr 18:28:33.568 # Redis is now ready to exit, bye bye...
_._
_.-``__ ''-._
_.-`` `. `_. ''-._ Redis 2.8.18 (00000000/0) 64 bit
.-`` .-```. ```\/ _.,_ ''-._
( ' , .-` | `, ) Running in stand alone mode
|`-._`-...-` __...-.``-._|'` _.-'| Port: 6379
| `-._ `._ / _.-' | PID: 206040
`-._ `-._ `-./ _.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' | http://redis.io
`-._ `-._`-.__.-'_.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' |
`-._ `-._`-.__.-'_.-' _.-'
`-._ `-.__.-' _.-'
`-._ _.-'
`-.__.-'
[206040] 19 Apr 18:28:33.580 # Server started, Redis version 2.8.18
[206040] 19 Apr 18:28:33.580 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
[206040] 19 Apr 18:28:33.580 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.
[206040] 19 Apr 18:28:33.580 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
[206040] 19 Apr 18:28:33.605 - Accepted 127.0.0.1:17454
[206040] 19 Apr 18:28:33.610 * DB loaded from disk: 0.030 seconds
[206040] 19 Apr 18:28:33.610 * The server is now ready to accept connections on port 6379
[206040] 19 Apr 18:28:33.610 - Client closed connection
[206040] 19 Apr 18:28:33.610 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:33.611 - 0 clients connected (0 slaves), 4798104 bytes in use
[206040] 19 Apr 18:28:33.671 - Accepted 172.16.186.205:16672
[206040] 19 Apr 18:28:34.341 - Accepted 172.16.186.203:18122
[206040] 19 Apr 18:28:34.342 * Slave 172.16.186.203:6379 asks for synchronization
[206040] 19 Apr 18:28:34.342 * Partial resynchronization not accepted: Runid mismatch (Client asked for '0c4731011b0b911b000c1d70fdc3f907f76ce180', I'm '2ae45a0020a36a175b290e23f04672c54fc7fdef')
[206040] 19 Apr 18:28:34.342 * Starting BGSAVE for SYNC with target: disk
[206040] 19 Apr 18:28:34.344 * Background saving started by pid 206049
[206049] 19 Apr 18:28:34.397 * DB saved on disk
[206049] 19 Apr 18:28:34.398 * RDB: 10 MB of memory used by copy-on-write
[206040] 19 Apr 18:28:34.413 * Background saving terminated with success
[206040] 19 Apr 18:28:34.436 * Synchronization with slave 172.16.186.203:6379 succeeded
[206040] 19 Apr 18:28:36.610 - Accepted 127.0.0.1:17513
[206040] 19 Apr 18:28:36.611 - Client closed connection
[206040] 19 Apr 18:28:36.615 - Accepted 172.16.186.205:11486
[206040] 19 Apr 18:28:36.615 - Client closed connection
[206040] 19 Apr 18:28:38.621 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:38.621 - 1 clients connected (1 slaves), 5889264 bytes in use
[206040] 19 Apr 18:28:43.627 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:43.627 - 1 clients connected (1 slaves), 5888208 bytes in use
[206040] 19 Apr 18:28:48.635 - DB 0: 13575 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:48.636 - 1 clients connected (1 slaves), 5888208 bytes in use
[206040] 19 Apr 18:28:53.577 - Client closed connection
[206040] 19 Apr 18:28:53.577 # Connection with slave 172.16.186.203:6379 lost.
[206040] 19 Apr 18:28:53.645 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:53.645 - 1 clients connected (0 slaves), 5870128 bytes in use
[206040] 19 Apr 18:28:57.632 - Accepted 172.16.186.203:18604
[206040] 19 Apr 18:28:57.632 * Slave 172.16.186.203:6379 asks for synchronization
[206040] 19 Apr 18:28:57.632 * Full resync requested by slave 172.16.186.203:6379
[206040] 19 Apr 18:28:57.633 * Starting BGSAVE for SYNC with target: disk
[206040] 19 Apr 18:28:57.634 * Background saving started by pid 206569
[206569] 19 Apr 18:28:57.690 * DB saved on disk
[206569] 19 Apr 18:28:57.691 * RDB: 12 MB of memory used by copy-on-write
[206040] 19 Apr 18:28:57.752 * Background saving terminated with success
[206040] 19 Apr 18:28:57.773 * Synchronization with slave 172.16.186.203:6379 succeeded
[206040] 19 Apr 18:28:58.653 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:28:58.653 - 1 clients connected (1 slaves), 5929296 bytes in use
[206040] 19 Apr 18:29:03.661 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:03.661 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:08.670 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:08.670 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:13.679 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:13.679 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:18.689 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:18.689 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:23.698 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:23.698 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:28.706 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:28.706 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:33.715 - DB 0: 13572 keys (0 volatile) in 16384 slots HT.
[206040] 19 Apr 18:29:33.715 - 1 clients connected (1 slaves), 5930992 bytes in use
[206040] 19 Apr 18:29:38.580 - Accepted 127.0.0.1:18538
[206040] 19 Apr 18:29:38.581 # User requested shutdown...
[206040] 19 Apr 18:29:38.581 * Saving the final RDB snapshot before exiting.
[206040] 19 Apr 18:29:38.636 * DB saved on disk
[206040] 19 Apr 18:29:38.636 * Removing the pid file.
[206040] 19 Apr 18:29:38.636 # Redis is now ready to exit, bye bye...
_._
_.-``__ ''-._
_.-`` `. `_. ''-._ Redis 2.8.18 (00000000/0) 64 bit
.-`` .-```. ```\/ _.,_ ''-._
( ' , .-` | `, ) Running in stand alone mode
|`-._`-...-` __...-.``-._|'` _.-'| Port: 6379
| `-._ `._ / _.-' | PID: 207901
`-._ `-._ `-./ _.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' | http://redis.io
`-._ `-._`-.__.-'_.-' _.-'
|`-._`-._ `-.__.-' _.-'_.-'|
| `-._`-._ _.-'_.-' |
`-._ `-._`-.__.-'_.-' _.-'
`-._ `-.__.-' _.-'
`-._ _.-'
`-.__.-'
[207901] 19 Apr 18:29:38.649 # Server started, Redis version 2.8.18
- shutdown 命令来自于 127.0.0.1 <-> 127.0.0.1 的 TCP 连接；
- redis 接收 shutdown 后会自行关闭 socket ，所以 TIME_WAIT 状态在 redis 侧；
- 每 55 秒左右 shutdown 一次；
问题三：终端设备通过 HTTP 协议经由 nginx 访问后端的 api 服务器时，TCP 连接行为诡异
... Apr 20 18:21:48 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:24:37 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:25:50 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:27:02 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:29:01 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:30:14 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:31:28 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:32:44 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:35:33 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:37:06 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:37:52 localhost ntpd_intres[1732]: host name not found: 0.centos.pool.ntp.org Apr 20 18:38:12 localhost ntpd_intres[1732]: host name not found: 1.centos.pool.ntp.org Apr 20 18:38:20 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:38:32 localhost ntpd_intres[1732]: host name not found: 2.centos.pool.ntp.org Apr 20 18:38:52 localhost ntpd_intres[1732]: host name not found: 3.centos.pool.ntp.org Apr 20 18:39:29 localhost kernel: possible SYN flooding on port 80. Sending cookies. Apr 20 18:40:43 localhost kernel: possible SYN flooding on port 80. Sending cookies. ...
问题四：