当一个 Bug 成了谜：一次诡异的变量为空导致脚本误删文件夹
ps 看到进程列表里有一堆一年前的 rsync 僵死进程，看着不爽，于是 kill 之，几分钟后，收到告警短信：xx 文件不存在了。上目录下一看，叉。。。确实整个目录被删了，查了下 scrt 操作日志以及脚本，没找到原因。无奈的是，该问题还不好重现，无法彻查原因，只好吸取教训：以后应当避免直接 rm -rf 删除文件夹，需要依赖前一个命令的状态，或者 find -mtime +n 依赖文件修改时间来删除。
下面贴的是本次 BUG 的全过程以及相关的 BUG 脚本，原因未知。。。
- 操作过程：
root@qwrz 10.14.18.21 16:17:30 /data1/logs/nginx >
ps -ef|grep nginx
root 742 655 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130116 [email protected]::js/66/
root 2802 2773 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130116 [email protected]::js/66/
root 5187 5102 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130116 [email protected]::js/66/
root 5642 5426 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 8074 8036 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 12710 12662 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 13197 2287 0 16:23 pts/0 00:00:00 grep --color nginx
root 13654 1 0 2013 ? 00:00:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx 13702 13654 5 Jul03 ? 2-09:12:30 nginx: worker process
nginx 13703 13654 5 Jul03 ? 2-09:13:17 nginx: worker process
nginx 13704 13654 5 Jul03 ? 2-09:10:52 nginx: worker process
nginx 13705 13654 5 Jul03 ? 2-09:11:34 nginx: worker process
nginx 13706 13654 5 Jul03 ? 2-09:11:25 nginx: worker process
nginx 13707 13654 5 Jul03 ? 2-09:13:02 nginx: worker process
nginx 13708 13654 5 Jul03 ? 2-09:10:22 nginx: worker process
nginx 13709 13654 5 Jul03 ? 2-09:11:42 nginx: worker process
root 16880 16781 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 18911 18848 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 21311 21238 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 22026 21774 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root 23962 23909 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root 26329 26270 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root@qwrz 10.14.18.21 16:23:53 /data1/logs/nginx >
ls -lrt /data1/logs/nginx/app/track/20130116
ls: /data1/logs/nginx/app/track/20130116: No such file or directory
root@qwrz 10.14.18.21 16:24:18 /data1/logs/nginx >
ps -ef|head -2
UID PID PPID C STIME TTY TIME CMD
root 1 0 0 2012 ? 00:03:33 init [3]
root@qwrz 10.14.18.21 16:26:00 /data1/logs/nginx >
kill -9 742
root@qwrz 10.14.18.21 16:26:09 /data1/logs/nginx >
ps -ef|grep nginx
root 2802 2773 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130116 [email protected]::js/66/
root 5187 5102 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130116 [email protected]::js/66/
root 5642 5426 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 8074 8036 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 12710 12662 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130228 [email protected]::js/66/
root 135ooxx 2287 0 16:26 pts/0 00:00:00 grep --color nginx
root 13654 1 0 2013 ? 00:00:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nginx 13702 13654 5 Jul03 ? 2-09:12:47 nginx: worker process
nginx 13703 13654 5 Jul03 ? 2-09:13:33 nginx: worker process
nginx 13704 13654 5 Jul03 ? 2-09:11:09 nginx: worker process
nginx 13705 13654 5 Jul03 ? 2-09:11:50 nginx: worker process
nginx 13706 13654 5 Jul03 ? 2-09:11:42 nginx: worker process
nginx 13707 13654 5 Jul03 ? 2-09:13:19 nginx: worker process
nginx 13708 13654 5 Jul03 ? 2-09:10:39 nginx: worker process
nginx 13709 13654 5 Jul03 ? 2-09:12:00 nginx: worker process
root 16880 16781 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 18911 18848 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 21311 21238 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130123 [email protected]::js/66/
root 22026 21774 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root 23962 23909 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root 26329 26270 0 2013 ? 00:00:00 rsync -vat /data1/logs/nginx/app/track/20130323 [email protected]::js/66/
root@qwrz 10.14.18.21 16:26:15 /data1/logs/nginx >
kill -9 2802
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 5187
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 5642
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 8074
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 12710
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 16880
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 18911
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 21311
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 22026
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 23962
root@qwrz 10.14.18.21 16:26:29 /data1/logs/nginx >
kill -9 26329
root@qwrz 10.14.18.21 16:26:30 /data1/logs/nginx >
kill -9 23962
root@qwrz 10.14.18.21 16:26:33 /data1/logs/nginx >
ps -ef|grep nginx
root 13591 8036 1 16:26 ? 00:00:00 rm -rf /data1/logs/nginx/m/track/
root 13592 5426 1 16:26 ? 00:00:00 rm -rf /data1/logs/nginx/m/track/
root 13600 12662 1 16:26 ? 00:00:00 rm -rf /data1/logs/nginx/m/track/
root 13654 1 0 2013 ? 00:00:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
root 13697 2287 0 16:26 pts/0 00:00:00 grep --color nginx
nginx 13702 13654 5 Jul03 ? 2-09:12:50 nginx: worker process
nginx 13703 13654 5 Jul03 ? 2-09:13:36 nginx: worker process
nginx 13704 13654 5 Jul03 ? 2-09:11:11 nginx: worker process
nginx 13705 13654 5 Jul03 ? 2-09:11:53 nginx: worker process
nginx 13706 13654 5 Jul03 ? 2-09:11:44 nginx: worker process
nginx 13707 13654 5 Jul03 ? 2-09:13:21 nginx: worker process
nginx 13708 13654 5 Jul03 ? 2-09:10:42 nginx: worker process
nginx 13709 13654 5 Jul03 ? 2-09:12:02 nginx: worker process
root@qwrz 10.14.18.21 17:38:35 /data1/logs/nginx/m >
cd track
root@qwrz 10.14.18.21 17:38:38 /data1/logs/nginx/m/track >
ls -lrt
total 0
root@qwrz 10.14.18.21 17:38:40 /data1/logs/nginx/m/track >
ls -lrt ../click
total 10562244
drwxr-xr-x 2 root root 4096 Aug 13 02:33 20140812
drwxr-xr-x 2 root root 4096 Aug 14 00:00 20140814
drwxr-xr-x 324 root root 12288 Aug 14 02:00 zhongzhuan
drwxr-xr-x 2 root root 4096 Aug 14 02:24 20140813
-rw-r--r-- 1 root root 10805140129 Aug 14 17:39 tracklog.ooxx.com.access.log
root@qwrz 10.14.18.21 22:56:57 ~ >
find / -name "*.sh" |xargs grep 'rm -rf /data1/logs/nginx/m/track'
/opt/bin/log.sh:rm -rf /data1/logs/nginx/m/track/$date2
root@qwrz 10.14.18.21 22:ooxx:05 ~ >
- bug 脚本：*/30 * * * * /opt/bin/log.sh >> /opt/bin/logs/log.log 2>&1
#!/bin/sh date1=`date -d "30 minutes ago" +%Y%m%d` date2=`date -d "3 days ago" +%Y%m%d` ############/data1/logs/nginx--rsync####################### rsync -vat /data1/logs/nginx/app/track/$date1 [email protected]::tracklog/allooxx-10.14.18.16/ rsync -vat /data1/logs/nginx/app/click/$date1 [email protected]::tracklog/appclickooxx-10.14.18.16/ #rsync -vat /data1/logs/nginx/pc/show/$date1 [email protected]::tracklog/showooxx-10.14.18.16/ #rsync -vat /data1/logs/nginx/pc/track/$date1 [email protected]::tracklog/pctrackooxx-10.14.18.16/ #rsync -vat /data1/logs/nginx/pc/click/$date1 [email protected]::tracklog/pcclickooxx-10.14.18.16/ rsync -vat /data1/logs/nginx/app/track/$date1 [email protected]::js/66/ rsync -vat /data1/logs/nginx/m/track/$date1 [email protected]::m/track-10.14.18.16/ ############/data1/logs/nginx--delete####################### rm -rf /data1/logs/nginx/app/track/$date2 rm -rf /data1/logs/nginx/pc/show/$date2 rm -rf /data1/logs/nginx/pc/track/$date2 rm -rf /data1/logs/nginx/pc/click/$date2 rm -rf /data1/logs/nginx/pc/mail/$date2 rm -rf /data1/logs/nginx/app/click/$date2 rm -rf /data1/logs/nginx/m/track/$date2 rm -rf /data1/logs/nginx/m/show/$date2 rm -rf /data1/logs/nginx/m/click/$date2 rm -rf /data1/logs/nginx/app/show/$date2
当一个 bug 成了谜
你不知道
他们为何出现
那次出现竟是他最后的一次。。。