前言
find作为最基础的bash shell命令之一我就不多做介绍了，由于每月压缩后的日志增长量已经超过20TB，和PB相比应该还不算很大，选择的日志归档同步方案是Facebook Scribe，之后或许会基于ELK或Grafana Loki搭建日志实时分析平台，不过眼下的问题还是想办法在没有商业化集中式存储和软件定义分布式存储的支持下，用比较简单粗暴的方法苦苦支撑，本文分享了一些简单的小技巧方便回顾和二次利用。
基于bash find命令执行log日志备份和清理日志简单实践
更新历史
2019年07月29日 - 初稿
阅读原文 - https://wsgzao.github.io/post/find/
扩展阅读
find
find
find - search for files in a directory hierarchy
https://linux.die.net/man/1/find
find 命令用来在指定目录下查找文件。任何位于参数之前的字符串都将被视为欲查找的目录名。如果使用该命令时，不设置任何参数，则 find 命令将在当前目录下查找子目录与文件。并且将查找到的子目录和文件全部进行显示。
https://man.linuxde.net/find
How to use
log archive
SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root
# For details see man 4 crontabs
# Example of job definition:
# .---------------- minute (0 - 59)
# | .------------- hour (0 - 23)
# | | .---------- day of month (1 - 31)
# | | | .------- month (1 - 12) OR jan,feb,mar,apr ...
# | | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
# | | | | |
# * * * * * user-name command to be executed
#zip
#15 3 * * * root cd /opt/sa_scripts/archive_from_date && bash run.sh /data/gop/live/primary/gop 2
15 3 * * * root bash /opt/sa_scripts/archive_from_date/one_run_2d.sh
log backup and cleanup
cd /data/scripts
./check_original_log.sh 2019-05
./move_backup.sh 2019-05
./check_backup_log.sh 2019-05
nohup sh rsync_backup_10-71-12-61.sh > log/2019-04.rsync.out 2>&1 &
nohup sh rsync_backup_10-71-14-132.sh > log/2019-05.rsync.out 2>&1 &
log archive
archive_from_date.sh
#!/bin/bash
#to compress all files like:
#auth_data-2017-01-11_00000
#auth_data-2017-01-11_00001
#auth_data-2017-01-11_00002
#auth_data-2017-01-11_00003
#auth_data-2017-01-12_00000
#auth_data-2017-01-12_00001
#
#by day till the specified date
if [ "$#" -lt 2 ]
then
echo "Usage: $0 path archive_days [trailing_digit]"
exit 1
fi
FILE_PATH=$1
ARCHIVE_DAYS=$2
#seems no need traling digit param here
TRAILING_DIGIT=$3
if [ -z "${TRAILING_DIGIT//}" ]
then
TRAILING_DIGIT=6
fi
start_date="2017-01-01"
end_date=`date`
end_date=`date -d -${ARCHIVE_DAYS}days +%Y-%m-%d`
echo "Scanning from $start_date to $end_date (not inclusive)"
echo "=================================="
if [[ "$end_date" < "$start_date" ]]
then
echo "Invalid end date: $end_date,it should be later than start date: ${start_date}. Exit..."
exit 1
fi
RESULT_PATH=/tmp/log_archive
if [ ! -d "$RESULT_PATH" ]
then
mkdir -p $RESULT_PATH
fi
cd $FILE_PATH
while [ "$start_date" != "$end_date" ]
do
if ls *${start_date}*[0-9] 1>/dev/null 2>&1
then
echo "Compressing: $start_date"
first_file=`ls *${start_date}*[0-9]|head -1`
file_name=${first_file%_*}
tar cvzf ${file_name}.tgz ${file_name}*[0-9]
# #Case 1: compare tar size with origin, uncomment if needed
# original_size=`ls -l *${start_date}*[0-9]|awk '{sum+=$5} END {print sum}'`
# weighted_size=$((original_size/20))
# tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
# echo $tar_size,$weighted_size
# if (( $tar_size < $weighted_size ))
# then
# echo "tar size: ${tar_size}; weighted size:${weighted_size}"
# echo "tar size too small; not deleting origin"
# echo "`pwd`: $file_name" >>/opt/sa_scripts/archive/result
# else
# echo "Done.Deleting origin."
# rm ${file_name}*[0-9]
# fi
# #End of Case 1
##############
#Case 2, compare tar size with 0
tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
if (( $tar_size > 0 ))
then
echo "Done.Deleting origin."
rm ${file_name}*[0-9]
else
echo "tar file size is ZERO!"
echo "`pwd`: $file_name" >>$RESULT_PATH
fi
#End of Case 2
fi
start_date=$(date -I -d "$start_date +1day")
done
archive_from_day.sh
#!/bin/bash
# bash one_day.sh /data/gop/live/primary/gop 2017-01-11
FILE_PATH=$1
ARCHIVE_DAYS=$2
start_date=$ARCHIVE_DAYS
end_date=$ARCHIVE_DAYS
echo "Scanning from $start_date to $end_date (not inclusive)"
echo "=================================="
RESULT_PATH=/tmp/log_archive
if [ ! -d "$RESULT_PATH" ]
then
mkdir -p $RESULT_PATH
fi
cd $FILE_PATH
if ls *${start_date}*[0-9] 1>/dev/null 2>&1
then
echo "Compressing: $start_date"
first_file=`ls *${start_date}*[0-9]|head -1`
file_name=${first_file%_*}
tar cvzf ${file_name}.tgz ${file_name}*[0-9]
#Case 2, compare tar size with 0
tar_size=`ls -l ${file_name}.tgz|awk '{print $5}'`
if (( $tar_size > 0 ))
then
echo "Done.Deleting origin."
rm ${file_name}*[0-9]
else
echo "tar file size is ZERO!"
echo "`pwd`: $file_name" >>$RESULT_PATH
fi
#End of Case 2
fi
run.sh
#!/bin/bash
PARENT_PATH=$1
ARCHIVE_DAY=$2
if [ "$#" -ne 2 ]
then
echo "Usage: $0 parent_path archive_day"
exit 1
fi
for folder in $(find $PARENT_PATH -type d -links 2)
do
echo "processing:${folder}..."
./archive_from_date.sh $folder $ARCHIVE_DAY
done
one_run_2d.sh
#!/bin/bash
end_date=`date -d -2days +%Y-%m-%d`
nohup bash /opt/sa_scripts/archive_from_date/one_run.sh /data/gop/live/primary/gop $end_date > /opt/sa_scripts/archive_from_date/logs/$end_date.log 2>&1 &
one_run.sh
#!/bin/bash
PARENT_PATH=$1
ARCHIVE_DAY=$2
if [ "$#" -ne 2 ]
then
echo "Usage: $0 parent_path archive_day"
exit 1
fi
for folder in $(find $PARENT_PATH -type d -links 2)
do
echo "processing:${folder}..."
/opt/sa_scripts/archive_from_date/day.sh $folder $ARCHIVE_DAY
done
log example
total 2197444
-rw-r--r-- 1 root root 35386495 Jun 3 21:20 gop-live-point_txn_aggregator-data-2019-06-01.tgz
-rw-r--r-- 1 root root 33273034 Jun 4 20:37 gop-live-point_txn_aggregator-data-2019-06-02.tgz
-rw-r--r-- 1 root root 33390884 Jun 5 20:23 gop-live-point_txn_aggregator-data-2019-06-03.tgz
-rw-r--r-- 1 root root 35553038 Jun 6 20:18 gop-live-point_txn_aggregator-data-2019-06-04.tgz
-rw-r--r-- 1 root root 35081679 Jun 7 19:16 gop-live-point_txn_aggregator-data-2019-06-05.tgz
-rw-r--r-- 1 root root 35397705 Jun 8 19:17 gop-live-point_txn_aggregator-data-2019-06-06.tgz
-rw-r--r-- 1 root root 36039754 Jun 9 18:55 gop-live-point_txn_aggregator-data-2019-06-07.tgz
-rw-r--r-- 1 root root 38596404 Jun 10 20:22 gop-live-point_txn_aggregator-data-2019-06-08.tgz
-rw-r--r-- 1 root root 38048982 Jun 11 20:26 gop-live-point_txn_aggregator-data-2019-06-09.tgz
-rw-r--r-- 1 root root 33026017 Jun 12 19:16 gop-live-point_txn_aggregator-data-2019-06-10.tgz
-rw-r--r-- 1 root root 29198108 Jun 13 19:56 gop-live-point_txn_aggregator-data-2019-06-11.tgz
-rw-r--r-- 1 root root 25771109 Jun 14 19:19 gop-live-point_txn_aggregator-data-2019-06-12.tgz
-rw-r--r-- 1 root root 26092930 Jun 15 19:09 gop-live-point_txn_aggregator-data-2019-06-13.tgz
-rw-r--r-- 1 root root 35349519 Jun 16 19:13 gop-live-point_txn_aggregator-data-2019-06-14.tgz
-rw-r--r-- 1 root root 31780817 Jun 17 21:27 gop-live-point_txn_aggregator-data-2019-06-15.tgz
-rw-r--r-- 1 root root 36549729 Jun 18 21:57 gop-live-point_txn_aggregator-data-2019-06-16.tgz
-rw-r--r-- 1 root root 26007332 Jun 19 19:58 gop-live-point_txn_aggregator-data-2019-06-17.tgz
-rw-r--r-- 1 root root 25114440 Jun 20 19:20 gop-live-point_txn_aggregator-data-2019-06-18.tgz
-rw-r--r-- 1 root root 22494565 Jun 21 19:25 gop-live-point_txn_aggregator-data-2019-06-19.tgz
-rw-r--r-- 1 root root 25132986 Jun 22 19:19 gop-live-point_txn_aggregator-data-2019-06-20.tgz
-rw-r--r-- 1 root root 30698833 Jun 23 20:08 gop-live-point_txn_aggregator-data-2019-06-21.tgz
-rw-r--r-- 1 root root 38639032 Jun 24 21:12 gop-live-point_txn_aggregator-data-2019-06-22.tgz
-rw-r--r-- 1 root root 31422932 Jun 25 21:27 gop-live-point_txn_aggregator-data-2019-06-23.tgz
-rw-r--r-- 1 root root 25262186 Jun 26 19:57 gop-live-point_txn_aggregator-data-2019-06-24.tgz
-rw-r--r-- 1 root root 25402079 Jun 27 19:30 gop-live-point_txn_aggregator-data-2019-06-25.tgz
-rw-r--r-- 1 root root 22227072 Jun 28 17:09 gop-live-point_txn_aggregator-data-2019-06-26.tgz
-rw-r--r-- 1 root root 26174268 Jun 29 19:59 gop-live-point_txn_aggregator-data-2019-06-27.tgz
-rw-r--r-- 1 root root 27484303 Jun 30 20:43 gop-live-point_txn_aggregator-data-2019-06-28.tgz
-rw-r--r-- 1 root root 25607704 Jul 1 21:37 gop-live-point_txn_aggregator-data-2019-06-29.tgz
-rw-r--r-- 1 root root 23545276 Jul 2 21:34 gop-live-point_txn_aggregator-data-2019-06-30.tgz
-rw-r--r-- 1 root root 28956480 Jul 3 21:03 gop-live-point_txn_aggregator-data-2019-07-01.tgz
-rw-r--r-- 1 root root 27250475 Jul 4 20:21 gop-live-point_txn_aggregator-data-2019-07-02.tgz
-rw-r--r-- 1 root root 26443417 Jul 5 20:56 gop-live-point_txn_aggregator-data-2019-07-03.tgz
-rw-r--r-- 1 root root 23445274 Jul 6 20:28 gop-live-point_txn_aggregator-data-2019-07-04.tgz
-rw-r--r-- 1 root root 35914567 Jul 7 21:01 gop-live-point_txn_aggregator-data-2019-07-05.tgz
-rw-r--r-- 1 root root 42759699 Jul 8 21:46 gop-live-point_txn_aggregator-data-2019-07-06.tgz
-rw-r--r-- 1 root root 41146298 Jul 9 23:28 gop-live-point_txn_aggregator-data-2019-07-07.tgz
-rw-r--r-- 1 root root 30890861 Jul 10 21:50 gop-live-point_txn_aggregator-data-2019-07-08.tgz
-rw-r--r-- 1 root root 29505781 Jul 11 22:16 gop-live-point_txn_aggregator-data-2019-07-09.tgz
-rw-r--r-- 1 root root 25130694 Jul 12 21:04 gop-live-point_txn_aggregator-data-2019-07-10.tgz
-rw-r--r-- 1 root root 30355468 Jul 13 21:44 gop-live-point_txn_aggregator-data-2019-07-11.tgz
-rw-r--r-- 1 root root 32142222 Jul 14 22:47 gop-live-point_txn_aggregator-data-2019-07-12.tgz
-rw-r--r-- 1 root root 30846192 Jul 15 22:54 gop-live-point_txn_aggregator-data-2019-07-13.tgz
-rw-r--r-- 1 root root 32435568 Jul 17 00:21 gop-live-point_txn_aggregator-data-2019-07-14.tgz
-rw-r--r-- 1 root root 24601862 Jul 17 23:54 gop-live-point_txn_aggregator-data-2019-07-15.tgz
-rw-r--r-- 1 root root 26708856 Jul 19 00:54 gop-live-point_txn_aggregator-data-2019-07-16.tgz
-rw-r--r-- 1 root root 21915660 Jul 19 23:23 gop-live-point_txn_aggregator-data-2019-07-17.tgz
-rw-r--r-- 1 root root 19324816 Jul 21 00:01 gop-live-point_txn_aggregator-data-2019-07-18.tgz
-rw-r--r-- 1 root root 29234368 Jul 22 00:47 gop-live-point_txn_aggregator-data-2019-07-19.tgz
-rw-r--r-- 1 root root 32138171 Jul 23 02:44 gop-live-point_txn_aggregator-data-2019-07-20.tgz
-rw-r--r-- 1 root root 31359175 Jul 24 04:05 gop-live-point_txn_aggregator-data-2019-07-21.tgz
-rw-r--r-- 1 root root 26123344 Jul 24 23:36 gop-live-point_txn_aggregator-data-2019-07-22.tgz
-rw-r--r-- 1 root root 25337107 Jul 26 00:10 gop-live-point_txn_aggregator-data-2019-07-23.tgz
-rw-r--r-- 1 root root 23113614 Jul 26 23:52 gop-live-point_txn_aggregator-data-2019-07-24.tgz
-rw-r--r-- 1 root root 22578291 Jul 28 00:37 gop-live-point_txn_aggregator-data-2019-07-25.tgz
-rw-r--r-- 1 root root 28572628 Jul 29 01:21 gop-live-point_txn_aggregator-data-2019-07-26.tgz
-rw-r--r-- 1 root root 29820440 Jul 30 01:18 gop-live-point_txn_aggregator-data-2019-07-27.tgz
-rw-r--r-- 1 root root 25214160 Jul 31 01:26 gop-live-point_txn_aggregator-data-2019-07-28.tgz
-rw-r--r-- 1 root root 20978337 Aug 1 00:27 gop-live-point_txn_aggregator-data-2019-07-29.tgz
-rw-r--r-- 1 root root 22192620 Aug 1 23:12 gop-live-point_txn_aggregator-data-2019-07-30.tgz
-rw-r--r-- 1 root root 6329205 Jul 31 00:59 gop-live-point_txn_aggregator-data-2019-07-31_00000
-rw-r--r-- 1 root root 5786176 Jul 31 02:00 gop-live-point_txn_aggregator-data-2019-07-31_00001
-rw-r--r-- 1 root root 5451181 Jul 31 03:00 gop-live-point_txn_aggregator-data-2019-07-31_00002
-rw-r--r-- 1 root root 5621760 Jul 31 04:00 gop-live-point_txn_aggregator-data-2019-07-31_00003
-rw-r--r-- 1 root root 5820921 Jul 31 05:00 gop-live-point_txn_aggregator-data-2019-07-31_00004
-rw-r--r-- 1 root root 6088472 Jul 31 05:59 gop-live-point_txn_aggregator-data-2019-07-31_00005
-rw-r--r-- 1 root root 6048663 Jul 31 06:59 gop-live-point_txn_aggregator-data-2019-07-31_00006
-rw-r--r-- 1 root root 6434107 Jul 31 08:00 gop-live-point_txn_aggregator-data-2019-07-31_00007
-rw-r--r-- 1 root root 6937321 Jul 31 09:00 gop-live-point_txn_aggregator-data-2019-07-31_00008
-rw-r--r-- 1 root root 6717315 Jul 31 09:59 gop-live-point_txn_aggregator-data-2019-07-31_00009
-rw-r--r-- 1 root root 6667836 Jul 31 10:59 gop-live-point_txn_aggregator-data-2019-07-31_00010
-rw-r--r-- 1 root root 6559363 Jul 31 12:00 gop-live-point_txn_aggregator-data-2019-07-31_00011
-rw-r--r-- 1 root root 6107832 Jul 31 12:59 gop-live-point_txn_aggregator-data-2019-07-31_00012
-rw-r--r-- 1 root root 6323896 Jul 31 13:59 gop-live-point_txn_aggregator-data-2019-07-31_00013
-rw-r--r-- 1 root root 6389052 Jul 31 15:00 gop-live-point_txn_aggregator-data-2019-07-31_00014
-rw-r--r-- 1 root root 6504740 Jul 31 16:00 gop-live-point_txn_aggregator-data-2019-07-31_00015
-rw-r--r-- 1 root root 6796378 Jul 31 17:00 gop-live-point_txn_aggregator-data-2019-07-31_00016
-rw-r--r-- 1 root root 7878303 Jul 31 18:00 gop-live-point_txn_aggregator-data-2019-07-31_00017
-rw-r--r-- 1 root root 8900771 Jul 31 19:00 gop-live-point_txn_aggregator-data-2019-07-31_00018
-rw-r--r-- 1 root root 9379681 Jul 31 20:00 gop-live-point_txn_aggregator-data-2019-07-31_00019
-rw-r--r-- 1 root root 10435828 Jul 31 21:00 gop-live-point_txn_aggregator-data-2019-07-31_00020
-rw-r--r-- 1 root root 10127803 Jul 31 22:00 gop-live-point_txn_aggregator-data-2019-07-31_00021
-rw-r--r-- 1 root root 8788741 Jul 31 22:59 gop-live-point_txn_aggregator-data-2019-07-31_00022
-rw-r--r-- 1 root root 7747234 Jul 31 23:59 gop-live-point_txn_aggregator-data-2019-07-31_00023
-rw-r--r-- 1 root root 6983387 Aug 1 00:59 gop-live-point_txn_aggregator-data-2019-08-01_00000
-rw-r--r-- 1 root root 6572435 Aug 1 02:00 gop-live-point_txn_aggregator-data-2019-08-01_00001
-rw-r--r-- 1 root root 5959884 Aug 1 03:00 gop-live-point_txn_aggregator-data-2019-08-01_00002
-rw-r--r-- 1 root root 5834298 Aug 1 04:00 gop-live-point_txn_aggregator-data-2019-08-01_00003
-rw-r--r-- 1 root root 6008950 Aug 1 04:59 gop-live-point_txn_aggregator-data-2019-08-01_00004
-rw-r--r-- 1 root root 6614511 Aug 1 06:00 gop-live-point_txn_aggregator-data-2019-08-01_00005
-rw-r--r-- 1 root root 6934464 Aug 1 07:00 gop-live-point_txn_aggregator-data-2019-08-01_00006
-rw-r--r-- 1 root root 7501841 Aug 1 08:00 gop-live-point_txn_aggregator-data-2019-08-01_00007
-rw-r--r-- 1 root root 8018389 Aug 1 09:00 gop-live-point_txn_aggregator-data-2019-08-01_00008
-rw-r--r-- 1 root root 8113113 Aug 1 10:00 gop-live-point_txn_aggregator-data-2019-08-01_00009
-rw-r--r-- 1 root root 8247397 Aug 1 11:00 gop-live-point_txn_aggregator-data-2019-08-01_00010
-rw-r--r-- 1 root root 9334051 Aug 1 12:00 gop-live-point_txn_aggregator-data-2019-08-01_00011
-rw-r--r-- 1 root root 8284898 Aug 1 12:59 gop-live-point_txn_aggregator-data-2019-08-01_00012
-rw-r--r-- 1 root root 9218796 Aug 1 14:00 gop-live-point_txn_aggregator-data-2019-08-01_00013
-rw-r--r-- 1 root root 7875013 Aug 1 15:00 gop-live-point_txn_aggregator-data-2019-08-01_00014
-rw-r--r-- 1 root root 4420827 Aug 1 16:00 gop-live-point_txn_aggregator-data-2019-08-01_00015
-rw-r--r-- 1 root root 7999875 Aug 1 16:59 gop-live-point_txn_aggregator-data-2019-08-01_00016
-rw-r--r-- 1 root root 9480688 Aug 1 18:00 gop-live-point_txn_aggregator-data-2019-08-01_00017
-rw-r--r-- 1 root root 10069991 Aug 1 19:00 gop-live-point_txn_aggregator-data-2019-08-01_00018
-rw-r--r-- 1 root root 11397317 Aug 1 20:00 gop-live-point_txn_aggregator-data-2019-08-01_00019
-rw-r--r-- 1 root root 12597957 Aug 1 21:00 gop-live-point_txn_aggregator-data-2019-08-01_00020
-rw-r--r-- 1 root root 11541451 Aug 1 22:00 gop-live-point_txn_aggregator-data-2019-08-01_00021
-rw-r--r-- 1 root root 10813577 Aug 1 23:00 gop-live-point_txn_aggregator-data-2019-08-01_00022
-rw-r--r-- 1 root root 9507499 Aug 2 00:00 gop-live-point_txn_aggregator-data-2019-08-01_00023
-rw-r--r-- 1 root root 8110763 Aug 2 01:00 gop-live-point_txn_aggregator-data-2019-08-02_00000
-rw-r--r-- 1 root root 7238339 Aug 2 02:00 gop-live-point_txn_aggregator-data-2019-08-02_00001
-rw-r--r-- 1 root root 6577030 Aug 2 02:59 gop-live-point_txn_aggregator-data-2019-08-02_00002
-rw-r--r-- 1 root root 6586196 Aug 2 04:00 gop-live-point_txn_aggregator-data-2019-08-02_00003
-rw-r--r-- 1 root root 6883540 Aug 2 05:00 gop-live-point_txn_aggregator-data-2019-08-02_00004
-rw-r--r-- 1 root root 7172237 Aug 2 06:00 gop-live-point_txn_aggregator-data-2019-08-02_00005
-rw-r--r-- 1 root root 7293336 Aug 2 07:00 gop-live-point_txn_aggregator-data-2019-08-02_00006
-rw-r--r-- 1 root root 7728992 Aug 2 08:00 gop-live-point_txn_aggregator-data-2019-08-02_00007
-rw-r--r-- 1 root root 8154425 Aug 2 09:00 gop-live-point_txn_aggregator-data-2019-08-02_00008
-rw-r--r-- 1 root root 8036346 Aug 2 09:59 gop-live-point_txn_aggregator-data-2019-08-02_00009
-rw-r--r-- 1 root root 7940669 Aug 2 11:00 gop-live-point_txn_aggregator-data-2019-08-02_00010
-rw-r--r-- 1 root root 7901900 Aug 2 12:00 gop-live-point_txn_aggregator-data-2019-08-02_00011
-rw-r--r-- 1 root root 8501400 Aug 2 13:00 gop-live-point_txn_aggregator-data-2019-08-02_00012
-rw-r--r-- 1 root root 8024730 Aug 2 13:59 gop-live-point_txn_aggregator-data-2019-08-02_00013
-rw-r--r-- 1 root root 8413981 Aug 2 14:59 gop-live-point_txn_aggregator-data-2019-08-02_00014
-rw-r--r-- 1 root root 5899646 Aug 2 15:50 gop-live-point_txn_aggregator-data-2019-08-02_00015
lrwxrwxrwx 1 root root 51 Aug 2 15:00 gop-live-point_txn_aggregator-data_current -> gop-live-point_txn_aggregator-data-2019-08-02_00015
backup and rsync scripts
check_original_log.sh
#!/bin/bash
var_date=$1
var_src="/data/gop/live/primary/gop"
find ${var_src} | grep -i ${var_date}
check_backup_log.sh
#!/bin/bash
var_date=$1
var_src="/data/backup/"
find ${var_src} | grep -i ${var_date}
move_backup.sh
#!/bin/bash
var_date=\$1
var_path="/data/gop/live/primary/gop"
var_str="/data/backup"
# analyze log
find ${var_path} | grep -i ${var_date} > log/${var_date}.ori.out
cp log/${var_date}.ori.out log/${var_date}.mod.out
sed -i "s:/data/gop/live/primary/gop:${var_str}:g" log/\${var_date}.mod.out
# merge move action
paste -d "|" log/${var_date}.ori.out log/${var_date}.mod.out > log/\${var_date}.out
# move files
for i in `cat log/${var_date}.out`
do
var_a=`echo ${i} | cut -f1 -d"|"`
var_b=`echo ${i} | cut -f2 -d"|"`
mv ${var_a} ${var_b}
done
rsync_backup.sh
#!/bin/bash
var_src="/data/backup/"
var_des="rsync://10.71.12.61:873/backup/"
rsync -aP ${var_src} ${var_des}
clean_backup_log.sh
#!/bin/bash
var_date=$1
var_src="/data/backup/"
find ${var_src} | grep -i ${var_date} | xargs rm -f
log out samples
# 2019-05.ori.out
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-26.tgz
# 2019-05.mod.out
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-29.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-23.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-30.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-27.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-28.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-24.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-25.tgz
/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-26.tgz
# 2019-05.out
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-25.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-27.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-23.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-30.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-31.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-28.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-24.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-26.tgz
/data/gop/live/primary/gop/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz|/data/backup/gop-live-gop_huawei_data_server-daemon/gop-live-gop_huawei_data_server-daemon-2019-05-29.tgz
/data/gop/live/primary/gop/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz|/data/backup/gop-live-gop_line_data_server-daemon/gop-live-gop_line_data_server-daemon-2019-05-31.tgz
# 2019-05.rsync.out
nohup: ignoring input
sending incremental file list
./
gop-live-account.garena.com-access/
gop-live-account.garena.com-access/gop-live-account.garena.com-access-2019-05-01.tgz
^M 32,768 0% 0.00kB/s 0:00:00 ^M 75,551,185 100% 188.53MB/s 0:00:00 (xfr#1, ir-chk=1053/1441)
gop-live-account.garena.com-access/gop-live-account.garena.com-access-2019-05-02.tgz
^M 32,768 0% 81.01kB/s 0:15:24 ^M 74,896,153 100% 82.01MB/s 0:00:00 (xfr#2, ir-chk=1052/1441)
log cleanup exclude list
/data/nc_backup
gop-live-tcp_server
gop-live-sso_website
gop-live-app_point
gop-live-gop_vk_data_server
gop-live-gop_huawei_data_server
gop-live-gop_line_data_server
gop-live-gop_google_data_server
gop-live-gop_fb_data_server
gop-live-gop_data_server
gop-live-general-api_server
gop-live-payment_center
gop-live-msdk_api
gop-live-cron_autofix
gop-live-shell_api_server
gop-live-api_server
gop-staging-payment_center
gop-staging-msdk_api
create python script to combine multiple lines
data=open("file").readlines()
for n,line in enumerate(data):
if line.startswith("line"):
data[n] = "\n"+line.rstrip()
else:
data[n]=line.rstrip()
print ('|'.join(data))
# simple example
Englist
中文
Englist|中文
# can change line.startwith
line1
text1
text2
text3
line2
something1
something2
line1|text1|text2|text3|
line2|something1|something2
use find to filter the log
# use find command filter directory
find /data/nc_backup -type d > log_source
find /data/nc_backup -type d | egrep -v "gop-live-tcp_server|gop-live-sso_website|gop-live-app_point|gop-live-gop_vk_data_server|gop-live-gop_huawei_data_server|gop-live-gop_line_data_server|gop-live-gop_google_data_server|gop-live-gop_fb_data_server|gop-live-gop_data_server|gop-live-general-api_server|gop-live-payment_center|gop-live-msdk_api|gop-live-cron_autofix|gop-live-shell_api_server|gop-live-api_server|gop-staging-payment_center|gop-staging-msdk_api" > log_target
# remove the fist line
sed -i '1d' log_source
sed -i '1d' log_target