一次DNS 故障引发的linux telnet 各端口缓慢的问题解决过程
昨天部署好了lvs+keepalived 并通过测试, 没有发现问题.今天上午忽然发现, 用ipvsadm �Cl 查看lvs信息,响应很慢, 然后去从LVS telnet 节点的22号端口, 发现特别慢.
开始我检查了一下keepalived.conf配置文件, 以为是同网段内多个lvs 设置,造成多播冲突,阻塞网络. 后来停止了lvs后故障依旧. 突然想到使用strace来分析, 一下找到了原因.
telnet 命令 调用了如下共享对象库及文件, 说明telnet 先做权限和安全检查(如selinux) , 再做解析, 顺序是 , 先从hosts文件中找, 如果有对应的条目, 可直接接续, 如果没有,再从resolve.conf中找, 最后才反应成真正的IP地址.
今天由于dns 202.96.209.5 故障, 所以连接其53端口有问题, 等待超时过了才能继续. 所以telnet有几秒钟的卡顿.
解决方案: 删除resolve中的记录或指向可用的dns.
/etc/ld.so.preload
/etc/ld.so.cache
/usr/lib64/libkrb4.so.2
/usr/lib64/libkrb4.so.2
/usr/lib64/libdes425.so.3
/usr/lib64/libkrb5.so.3
/usr/lib64/libk5crypto.so.3
/lib64/libcom_err.so.2
/usr/lib64/libkrb5support.so.0
/lib64/libkeyutils.so.1
/lib64/libresolv.so.2
/usr/lib64/libncurses.so.5
/lib64/libselinux.so.1
/lib64/libdl.so.2
/lib64/libc.so.6
/lib64/libsepol.so.1
/etc/selinux/config
/etc/nsswitch.conf
/etc/resolv.conf
[root@lvs1 ~]# stracetelnet 10.67.130.232 22
execve("/usr/kerberos/bin/telnet",["telnet", "10.67.130.232", "22"], [/* 22 vars*/]) = 0
brk(0) =0x2b867de32000
………………………………………………..
open("/etc/resolv.conf",O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=24,...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000
read(3, "nameserver 202.96.209.5\n", 4096) = 24
read(3, "", 4096) = 0
close(3) = 0
munmap(0x2b8670400000, 4096) = 0
uname({sys="Linux",node="lvs1", ...}) = 0
open("/etc/host.conf", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=0,...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000
read(3, "", 4096) = 0
close(3) = 0
munmap(0x2b8670400000, 4096) = 0
open("/etc/hosts", O_RDONLY) = 3
fcntl(3, F_GETFD) = 0
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
fstat(3, {st_mode=S_IFREG|0644,st_size=168, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000
read(3, "# Do not remove the followingli"..., 4096) = 168
read(3, "", 4096) = 0
close(3) = 0
munmap(0x2b8670400000, 4096) = 0
open("/etc/ld.so.cache",O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644,st_size=96055, ...}) = 0
mmap(NULL, 96055, PROT_READ, MAP_PRIVATE,3, 0) = 0x2b8670400000
close(3) = 0
open("/lib64/libnss_dns.so.2",O_RDONLY) = 3
read(3,"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\17\0\0\0\0\0\0"...,832) = 832
fstat(3, {st_mode=S_IFREG|0755,st_size=23736, ...}) = 0
mmap(NULL, 2113792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,3, 0) = 0x2b8672527000
mprotect(0x2b867252b000, 2093056,PROT_NONE) = 0
mmap(0x2b867272a000, 8192,PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) =0x2b867272a000
close(3) = 0
mprotect(0x2b867272a000, 4096, PROT_READ) =0
munmap(0x2b8670400000, 96055) = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 3
connect(3, {sa_family=AF_INET, sin_port=htons(53),sin_addr=inet_addr("202.96.209.5")}, 28) = 0
fcntl(3, F_GETFL) = 0x2 (flags O_RDWR)
fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
poll([{fd=3, events=POLLOUT}], 1, 0) = 1 ([{fd=3, revents=POLLOUT}])
sendto(3,"\255a\1\0\0\1\0\0\0\0\0\0\003232\003130\00267\00210\7in-ad"..., 44,MSG_NOSIGNAL, NULL, 0) = 44
poll([{fd=3, events=POLLIN}], 1, 5000) = 0 (Timeout)
poll([{fd=3, events=POLLOUT}], 1, 0) = 1 ([{fd=3, revents=POLLOUT}])
sendto(3,"\255a\1\0\0\1\0\0\0\0\0\0\003232\003130\00267\00210\7in-ad"..., 44,MSG_NOSIGNAL, NULL, 0) = 44
poll([{fd=3,events=POLLIN}], 1, 5000) = 0 (Timeout)
close(3) = 0
fstat(1, {st_mode=S_IFCHR|0620,st_rdev=makedev(136, 2), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b8670400000
write(1, "Trying 10.67.130.232...\r\n", 25Trying10.67.130.232...
) = 25
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) =3
setsockopt(3, SOL_IP, IP_TOS, [16], 4) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(22),sin_addr=inet_addr("10.67.130.232")}, 16) = 0
open("/proc/filesystems",O_RDONLY) = 4
read(4,"nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 4095) = 331
close(4) = 0
open("/root/.telnetrc",O_RDONLY) = -1 ENOENT (No such fileor directory)
open("/proc/filesystems",O_RDONLY) = 4
read(4,"nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 4095) = 331
close(4) = 0
write(1, "Connected to 10.67.130.232(10.6"..., 45Connected to 10.67.130.232 (10.67.130.232).
) = 45