shell使用脚本——自动处理重复数据

#!/bin/bash
Usage="Usage : $0 (arg1:the data file; arg2:the duplicate content file; arg3:the output file"
if [ $# -lt 3 ];then
  echo $Usage
  exit 1
fi
line_num=`cat $1 | wc -l`
for (( i=1; i<=$line_num; i++))
do
  line=`awk "NR==$i{print}" $1`
  key1=`echo $line | awk -F " " '{print $1}'`
  key2=`echo $line | awk -F " " '{print $2}'`
  key3=`echo $line | awk -F " " '{print $3}'`
  content=`sed "s/key1/$key1/g;s/key2/$key2/g;s/key3/$key3/g" $2`
  echo $content >> $3
done
exit 0

你可能感兴趣的:(大数据之Linux)