Linux Shell 实现网页爬虫

#!/bin/bash

cat ./cidlist | while read CID #从文件cidlist读取CID

do
{
echo $CID
        lynx -dump http://cheminfov.informatics.indiana.edu/rest/Chem2Bio2RDF/slap/cid=$CID  | grep "see paths" > target#抓取网页并获得包含"see paths"的行，写入target文件

        cat target | grep -Eo "[A-Z]+[A-Z|0-9]*" > ./slap/bak$CID#正则表达式提取所需要的字段

        cat ./slap/bak$CID | awk '{ print VAR,$0 } ' VAR=$CID > ./slap/$CID#格式化输出到slap文件夹下

        rm -r ./slap/bak$CID
echo $CID"is done!"
}

done