网站图片抓取

阅读更多

代码没有优化，仅仅作为大概的流程记录：

var http = require('http')
  , fs = require('fs');
var urlTpl = 'http://somedomain/loadmore.ashx?page={PAGE}&price=0&brandid=0&country=0&deliveryCapacity=0&level=0&struct=0&seat=0';

var counter = 0;
var page = 1,
    saveImage = function(res,name){
        var imagedata = '';
        res.setEncoding('binary');

        res.on('data', function(chunk){
            imagedata += chunk
        });

        res.on('end', function(){
            fs.writeFile("D:/ziliao/Node/pic/" + name + ".jpg", imagedata, 'binary', function(err){
                if (err) throw err
                console.log('File saved.')
            })
        });

    };

var reg = /src="(.*?)".*?title="(.*?)"/gi;
var reconvert = function(str){
    return str.replace(/(\\u)(\w{4}|\w{2})/gi, function($0,$1,$2){
        return String.fromCharCode(parseInt($2,16));
    }); 
} 
while(page < 26){
    url = urlTpl.replace(/\{PAGE\}/,page);
    var req = http.get(url,function(res){
         res.setEncoding('utf-8');

         var rs = null ,str = '';

         res.on('data', function (chunk) {
             str += chunk;
         });
         res.on('end',function(){
             while(true){
               //测试用，我们只要50张图片
               if(counter++ > 50) {break;}
               
               rs = reg.exec(str);
               if(rs === null){ break;}
              (
                function(url,name){
                  http.get(url,function(res){
                        //name  =  reconvert(name);
                        saveImage(res,reconvert(name));
                  });
                }
              )(rs[1],rs[2]);
            } 

         })
     });

     page++;
}