Linux strace命令的妙用

由于所使用Linux系统进行了裁减，出现了无法解析域名的情况

并且由于tcpdump和service的缺失，对于定位问题带来了一定的麻烦，不过还好strace命令可以用，下面就以解决dns无法解析为例，来说明strace妙用。

strace命令是一个集诊断、调试、统计与一体的工具，我们可以使用strace对应用的系统调用和信号传递的跟踪结果来对应用进行分析，以达到解决问题或者是了解应用工作过程的目的。strace的最简单的用法就是执行一个指定的命令，在指定的命令结束之后它也就退出了。在命令执行的过程中，strace会记录和解析命令进程的所有系统调用以及这个进程所接收到的所有的信号值。

1.strace+shell命令

strace nslookup www.baidu.com

/etc/waf/sites/2/nginx # strace nslookup www.baidu.com
execve("/usr/bin/nslookup", ["nslookup", "www.baidu.com"], [/* 19 vars */]) = 0
brk(0)                                  = 0x711388
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed7d000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/usr/local/lib/tls/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/lib/tls/x86_64", 0x7fff176a1290) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/tls/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/lib/tls", 0x7fff176a1290) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/lib/x86_64", 0x7fff176a1290) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/local/lib", {st_mode=S_IFDIR|0777, st_size=2800, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/tls/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/tls/x86_64", 0x7fff176a1290) = -1 ENOENT (No such file or directory)
open("/lib64/tls/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/tls", 0x7fff176a1290)      = -1 ENOENT (No such file or directory)
open("/lib64/x86_64/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/lib64/x86_64", 0x7fff176a1290)   = -1 ENOENT (No such file or directory)
open("/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 7
read(7, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260T\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0777, st_size=1127119, ...}) = 0
mmap(NULL, 3150168, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x7f3aee85c000
mprotect(0x7f3aee95d000, 2093056, PROT_NONE) = 0
mmap(0x7f3aeeb5c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x100000) = 0x7f3aeeb5c000
close(7)                                = 0
open("/usr/local/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 7
read(7, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\34\2\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0777, st_size=2063482, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed7c000
mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x7f3aee49b000
mprotect(0x7f3aee651000, 2097152, PROT_NONE) = 0
mmap(0x7f3aee851000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x1b6000) = 0x7f3aee851000
mmap(0x7f3aee857000, 16960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3aee857000
close(7)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed7a000
arch_prctl(ARCH_SET_FS, 0x7f3aeed7a740) = 0
mprotect(0x7f3aee851000, 16384, PROT_READ) = 0
mprotect(0x7f3aeeb5c000, 4096, PROT_READ) = 0
mprotect(0x70d000, 4096, PROT_READ)     = 0
mprotect(0x7f3aeed7e000, 4096, PROT_READ) = 0
getpid()                                = 6853
getuid()                                = 0
brk(0)                                  = 0x711388
brk(0x732388)                           = 0x732388
brk(0)                                  = 0x732388
brk(0x733000)                           = 0x733000
open("/etc/resolv.conf", O_RDONLY|O_CLOEXEC) = 7
fstat(7, {st_mode=S_IFREG|0644, st_size=52, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed79000
read(7, "options timeout:5 attempts:2\nnam"..., 4096) = 52
read(7, "", 4096)                       = 0
close(7)                                = 0
munmap(0x7f3aeed79000, 4096)            = 0
brk(0)                                  = 0x733000
brk(0)                                  = 0x733000
brk(0x732000)                           = 0x732000
brk(0)                                  = 0x732000
uname({sys="Linux", node="vefa", ...})  = 0
fstat(1, {st_mode=S_IFCHR|0622, st_rdev=makedev(3, 3), ...}) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed79000
write(1, "Server: 10.188.7.10\n", 23Server:    10.188.7.10
) = 23
socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 7
connect(7, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(7)                                = 0
socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 7
connect(7, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(7)                                = 0
open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/local/lib/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/lib64/tls/x86_64/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/tls/x86_64", 0x7fff176a0860) = -1 ENOENT (No such file or directory)
open("/usr/lib64/tls/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/tls", 0x7fff176a0860)  = -1 ENOENT (No such file or directory)
open("/usr/lib64/x86_64/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/x86_64", 0x7fff176a0860) = -1 ENOENT (No such file or directory)
open("/usr/lib64/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64", {st_mode=S_IFDIR|0777, st_size=0, ...}) = 0
open("/usr/local/lib/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 7
read(7, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\"\0\0\0\0\0\0"..., 832) = 832
fstat(7, {st_mode=S_IFREG|0755, st_size=57577, ...}) = 0
mmap(NULL, 2144360, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x7f3aee28f000
mprotect(0x7f3aee29a000, 2093056, PROT_NONE) = 0
mmap(0x7f3aee499000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0xa000) = 0x7f3aee499000
close(7)                                = 0
mprotect(0x7f3aee499000, 4096, PROT_READ) = 0
open("/etc/host.conf", O_RDONLY|O_CLOEXEC) = 7
fstat(7, {st_mode=S_IFREG|0777, st_size=17, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed78000
read(7, "order hosts,bind\n", 1024)     = 17
read(7, "", 1024)                       = 0
close(7)                                = 0
munmap(0x7f3aeed78000, 4096)            = 0
open("/etc/hosts", O_RDONLY|O_CLOEXEC)  = 7
fstat(7, {st_mode=S_IFREG|0777, st_size=20, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed78000
read(7, "127.0.0.1\tlocalhost\n", 1024) = 20
read(7, "", 1024)                       = 0
close(7)                                = 0
munmap(0x7f3aeed78000, 4096)            = 0
write(1, "Address 1: 10.188.7.10\n", 23Address 1: 10.188.7.10
) = 23
write(1, "\n", 1
)                       = 1
open("/etc/hosts", O_RDONLY|O_CLOEXEC)  = 7
fstat(7, {st_mode=S_IFREG|0777, st_size=20, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3aeed78000
read(7, "127.0.0.1\tlocalhost\n", 1024) = 20
read(7, "", 1024)                       = 0
close(7)                                = 0
munmap(0x7f3aeed78000, 4096)            = 0
write(2, "nslookup: can't resolve 'www.bai"..., 40nslookup: can't resolve 'www.baidu.com' ) = 40 exit_group(1) = ?

从上面看到open(“/usr/lib64/libnss_dns.so.2”, O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)，说明libnss_dns.so.2缺失。通过查资料找到dns解析主要依赖三个库libnss_dns.so.2 libnss_files.so.2 libresolv.so.2，在本地查看独缺libnss_dns.so.2，故从网络上下载了libnss_dns-2.17.so。

2.ln -s软链接

ln -s libnss_dns-2.17.so libnss_dns.so.2
由于nslookup命令实际访问的文件是libnss_dns.so.2，所以libnss_dns-2.17.so下载下来还不能直接用，需要做个软链接，具体什么是软链接可参考http://www.cnblogs.com/joeblackzqq/archive/2011/03/20/1989625.html

3.nslookup www.baidu.com

重新运行nslooup www.baidu.com，此时应能正确解析域名。

/usr/lib64 # nslookup www.baidu.com
Server:    10.188.7.10
Address 1: 10.188.7.10 

Name:      www.baidu.com
Address 1: 61.135.169.125
Address 2: 61.135.169.121

4.其他情况

1. 如还不能解析，请检查/etc/resolv.conf文件，如果设置了timeout和attempts，查看其数值，将其适当增大。
2. 查看环境变量或重新strace nslookup www.baidu.com确保动态库路径正确，或继续排查其他问题。