将nginx日志中的蜘蛛记录删除掉

#!/bin/sh
if [ -r $1 ]; then
    #Delete Baiduspider
    baidu=$(grep -c 'Baiduspider' $1)
    if [ $baidu -gt 0 ]; then
        sed -i '/Baiduspider/d' $1
        echo "match Baiduspider $baidu line."
    else
        echo "Baiduspider not found."
    fi
    #Delete YandexBot
    yandex=$(grep -c 'YandexBot' $1)
    if [ $yandex -gt 0 ]; then
        sed -i '/YandexBot/d' $1
        echo "match YandexBot $yandex line."
    else
        echo "YandexBot not found."
    fi
    #Delete bingbot
    bing=$(grep -c 'bingbot' $1)
    if [ $bing -gt 0 ]; then
        sed -i '/bingbot/d' $1
        echo "match bingbot $bing line."
    else
        echo "bingbot not found."
    fi
    #Delete EasouSpider
    easou=$(grep -c 'EasouSpider' $1)
    if [ $easou -gt 0 ]; then
        sed -i '/EasouSpider/d' $1
        echo "match EasouSpider $easou line."
    else
        echo "EasouSpider not found."
    fi
    #Delete Googlebot
    Googlebot=$(grep -c 'Googlebot' $1)
    if [ $Googlebot -gt 0 ]; then
        sed -i '/Googlebot/d' $1
        echo "match Googlebot $Googlebot line."
    else
        echo "Googlebot not found."
    fi
    #Delete Mediapartners-Google
    mgoogle=$(grep -c 'Mediapartners-Google' $1)
    if [ $mgoogle -gt 0 ]; then
        sed -i '/Mediapartners-Google/d' $1
        echo "match Mediapartners-Google $mgoogle line."
    else
        echo "Mediapartners-Google not found."
    fi
    #Delete Yahoo
    yahoo=$(grep -c 'Yahoo' $1)
    if [ $yahoo -gt 0 ]; then
        sed -i '/Yahoo/d' $1
        echo "match Yahoo $yahoo line."
    else
        echo "Yahoo not found."
    fi
    #Delete YoudaoBot
    youdao=$(grep -c 'YoudaoBot' $1)
    if [ $youdao -gt 0 ]; then
        sed -i '/YoudaoBot/d' $1
        echo "match YoudaoBot $youdao line."
    else
        echo "YoudaoBot not found."
    fi
    #Delete XoviBot
    xovi=$(grep -c 'XoviBot' $1)
    if [ $xovi -gt 0 ]; then
        sed -i '/XoviBot/d' $1
        echo "match XoviBot $xovi line."
    else
        echo "XoviBot not found."
    fi
    #Delete MSNBot
    msn=$(grep -c 'MSNBot' $1)
    if [ $msn -gt 0 ]; then
        sed -i '/MSNBot/d' $1
        echo "match MSNBot $msn line."
    else
        echo "MSNBot not found."
    fi
    #Delete Sogou web spider
    sogou=$(grep -c 'Sogou web spider' $1)
    if [ $sogou -gt 0 ]; then
        sed -i '/Sogou web spider/d' $1
        echo "match Sogou web spider $sogou line."
    else
        echo "Sogou web spider not found."
    fi
    #Delete JikeSpider
    jike=$(grep -c 'JikeSpider' $1)
    if [ $jike -gt 0 ]; then
        sed -i '/JikeSpider/d' $1
        echo "match JikeSpider $jike line."
    else
        echo "JikeSpider not found."
    fi
    #Delete proximic
    proximic=$(grep -c 'proximic' $1)
    if [ $proximic -gt 0 ]; then
        sed -i '/proximic/d' $1
        echo "match proximic $proximic line."
    else
        echo "proximic not found."
    fi
    #Delete 360Spider
    so360=$(grep -c '360Spider' $1)
    if [ $so360 -gt 0 ]; then
        sed -i '/360Spider/d' $1
        echo "match 360Spider $so360 line."
    else
        echo "360Spider not found."
    fi
    #Delete 360spider-image
    so360img=$(grep -c '360spider-image' $1)
    if [ $so360img -gt 0 ]; then
        sed -i '/360spider-image/d' $1
        echo "match 360spider-image $so360img line."
    else
        echo "360spider-image not found."
    fi
    #Delete YYSpider
    yy=$(grep -c 'YYSpider' $1)
    if [ $yy -gt 0 ]; then
        sed -i '/YYSpider/d' $1
        echo "match YYSpider $yy line."
    else
        echo "YYSpider not found."
    fi
    #Delete AhrefsBot
    ahrefs=$(grep -c 'AhrefsBot' $1)
    if [ $ahrefs -gt 0 ]; then
        sed -i '/AhrefsBot/d' $1
        echo "match AhrefsBot $ahrefs line."
    else
        echo "AhrefsBot not found."
    fi
    #Delete msnbot
    msnbot=$(grep -c 'msnbot' $1)
    if [ $msnbot -gt 0 ]; then
        sed -i '/msnbot/d' $1
        echo "match msnbot $msnbot line."
    else
        echo "msnbot not found."
    fi
    #Delete YisouSpider
    yisou=$(grep -c 'YisouSpider' $1)
    if [ $yisou -gt 0 ]; then
        sed -i '/YisouSpider/d' $1
        echo "match YisouSpider $yisou line."
    else
        echo "YisouSpider not found."
    fi
    #Delete Facebot
    facebot=$(grep -c 'Facebot' $1)
    if [ $facebot -gt 0 ]; then
        sed -i '/Facebot/d' $1
        echo "match Facebot $facebot line."
    else
        echo "Facebot not found."
    fi
    #Delete GrapeshotCrawler
    grapeshot=$(grep -c 'GrapeshotCrawler' $1)
    if [ $grapeshot -gt 0 ]; then
        sed -i '/GrapeshotCrawler/d' $1
        echo "match GrapeshotCrawler $grapeshot line."
    else
        echo "GrapeshotCrawler not found."
    fi
    #Delete WeSEE
    wesee=$(grep -c 'WeSEE' $1)
    if [ $wesee -gt 0 ]; then
        sed -i '/WeSEE/d' $1
        echo "match WeSEE $wesee line."
    else
        echo "WeSEE not found."
    fi
    #Delete Sogou inst spider
    sogou_inst=$(grep -c 'Sogou inst spider' $1)
    if [ $sogou_inst -gt 0 ]; then
        sed -i '/Sogou inst spider/d' $1
        echo "match Sogou inst spider $sogou_inst line."
    else
        echo "Sogou inst spider not found."
    fi
    #Delete WebZIP/x.x
    webzip=$(grep -c 'www.spidersoft.com' $1)
    if [ $webzip -gt 0 ]; then
        sed -i '/www.spidersoft.com/d' $1
        echo "match WebZIP/x.x $webzip line."
    else
        echo "WebZIP/x.x not found."
    fi
    #Delete baidu mobile spider
    bms=$(grep -c 'www.baidu.com\/search\/spider.html' $1)
    if [ $bms -gt 0 ]; then
        sed -i '/www.baidu.com\/search\/spider.html/d' $1
        echo "match baidu mobile spider $bms line."
    else
        echo "baidu mobile spider not found."
    fi
    #Delete ChinasoSpider
    chinaso=$(grep -c 'ChinasoSpider' $1)
    if [ $chinaso -gt 0 ]; then
        sed -i '/ChinasoSpider/d' $1
        echo "match ChinasoSpider $chinaso line."
    else
        echo "ChinasoSpider not found."
    fi
    #Delete www.xxx.com
    xxx=$(grep -c 'www.xxx.com' $1)
    if [ $xxx -gt 0 ]; then
        sed -i '/www.xxx.com/d' $1
        echo "match www.xxx.com $xxx line."
    else
        echo "www.xxx.com not found."
    fi
    #Delete SEOENGWorldBot
    seoeng=$(grep -c 'SEOENGWorldBot' $1)
    if [ $seoeng -gt 0 ]; then
        sed -i '/SEOENGWorldBot/d' $1
        echo "match SEOENGWorldBot $seoeng line."
    else
        echo "SEOENGWorldBot not found."
    fi
    #Delete Mail.RU_Bot
    mailru=$(grep -c 'Mail.RU_Bot' $1)
    if [ $mailru -gt 0 ]; then
        sed -i '/Mail.RU_Bot/d' $1
        echo "match Mail.RU_Bot $mailru line."
    else
        echo "Mail.RU_Bot not found."
    fi
    #Delete Girafabot
    gira=$(grep -c 'Girafabot' $1)
    if [ $gira -gt 0 ]; then
        sed -i '/Girafabot/d' $1
        echo "match Girafabot $gira line."
    else
        echo "Girafabot not found."
    fi
    #Delete DotBot
    dotBot=$(grep -c 'DotBot' $1)
    if [ $dotBot -gt 0 ]; then
        sed -i '/DotBot/d' $1
        echo "match DotBot $dotBot line."
    else
        echo "DotBot not found."
    fi
    #Delete BLEXBot
    BLEXBot=$(grep -c 'BLEXBot' $1)
    if [ $BLEXBot -gt 0 ]; then
        sed -i '/BLEXBot/d' $1
        echo "match BLEXBot $BLEXBot line."
    else
        echo "BLEXBot not found."
    fi
else
    echo "please, input your log file"
fi

你可能感兴趣的:(spider,蜘蛛,robots.txt)