检测hdfs 文件是否存在并获取文件列表、大小、创建时间的shell开发
命名 check_hdfs_dir.sh
运行：
sh check_hdfs_dir.sh /usr/data/000/
代码如下：
#!/bin/bash
lujing=$1
echo $lujing
command="hadoop fsck "${lujing}" > hadoop_fsck.txt"
eval $command
val=`awk '/Total size/ {print $3}' hadoop_fsck.txt`
#val=97987999
val_m=$[1024*1024]
val_k=$[1024]
val_g=$[1024*1024*1024]
val_t=$[1024*1024*1024*1024]
if [ `expr $val / $val_t` -gt 1 ]
then
echo "文件大小`expr $val / $val_t`t"
elif [ `expr $val / $val_g` -gt 1 ]
then
echo "文件大小`expr $val / $val_g`g"
elif [ `expr $val / $val_m` -gt 1 ]
then
echo "文件大小`expr $val / $val_m`m"
elif [ `expr $val / $val_k` -gt 1 ]
then
echo "文件大小`expr $val / $val_k`k"
fi
val1=`awk '/Total dirs/ {print $3}' hadoop_fsck.txt`
echo "文件夹数aa:$val1"
val2=`awk '/Total files/ {print $3}' hadoop_fsck.txt`
echo "文件数bb:$val2"
val3=`awk '/Total blocks (validated)/ {print $3}' hadoop_fsck.txt`
echo "总块数cc:$val3"
val_k=$[1024]
val_g=$[1024*1024*1024]
val_t=$[1024*1024*1024*1024]
val_m=$[1024*1024]
command="hadoop fs -ls "${lujing}" > hadoop_ls.txt"
eval $command
command="grep "${lujing}" hadoop_ls.txt | head -1 | awk '{print \$6}'"
echo "----****"
echo $command
chuangjianshijian=`eval $command`
echo "创建时间ddd=${chuangjianshijian}"
while read line
do
#if [true]
if [ ${line:0:5} == "Found" ]
then
echo "首行略过"
else
echo $line | awk -F ' ' '{print $5,$6,$7,$8}' | while read daxiao day time mulu
do
echo daxiao=$daxiao day=$day time=$time mulu=$mulu
length=${#lujing}
#echo -n ${lujing} | wc -c
file_name=${mulu:length}
echo 文件名=$file_name
#command="hadoop fs -cat hdfs://10.87.49.221:8020"${mulu}" | wc -l | sed -n '1p'"
command="hadoop fs -cat "${mulu}" | wc -l | sed -n '1p'"
hangshu=`eval $command`
#hangshu = $(hadoop fs -cat $mulu | wc -l)
echo "行数:$hangshu"
echo length=$length
val=$daxiao
if [ `expr $val / $val_t` -gt 1 ]
then
echo "文件大小`expr $val / $val_t`t"
elif [ `expr $val / $val_g` -gt 1 ]
then
echo "文件大小`expr $val / $val_g`g"
elif [ `expr $val / $val_m` -gt 1 ]
then
echo "文件大小`expr $val / $val_m`m"
elif [ `expr $val / $val_k` -gt 1 ]
then
echo "文件大小`expr $val / $val_k`k"
fi
echo "上传时间:day:$day,time:$time"
done
fi
done < hadoop_ls.txt