爬取网站的图片之一

阅读更多

    var http = require('http');  
    var fs = require('fs');
    var request = require('request');

function base64_decode (data) {
    var b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
    var o1, o2, o3, h1, h2, h3, h4, bits, i = 0, ac = 0, dec = "", tmp_arr = [];
    if (!data) {return data;}
    data += '';
    do { 
        h1 = b64.indexOf(data.charAt(i++));
        h2 = b64.indexOf(data.charAt(i++));
        h3 = b64.indexOf(data.charAt(i++));
        h4 = b64.indexOf(data.charAt(i++));
        bits = h1<<18 | h2<<12 | h3<<6 | h4;
        o1 = bits>>16 & 0xff;
        o2 = bits>>8 & 0xff;
        o3 = bits & 0xff;
        if (h3 == 64) {
            tmp_arr[ac++] = String.fromCharCode(o1);
        } else if (h4 == 64) {
            tmp_arr[ac++] = String.fromCharCode(o1, o2);
        } else {
            tmp_arr[ac++] = String.fromCharCode(o1, o2, o3);
        }
    } while (i < data.length);
    dec = tmp_arr.join('');
    dec = utf8_decode(dec);
    return dec;
}
function base64_encode (data) {
    var b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
    var o1, o2, o3, h1, h2, h3, h4, bits, i = 0, ac = 0, enc="", tmp_arr = [];
    if (!data){return data;}
    data = utf8_encode(data+'');
    do {
        o1 = data.charCodeAt(i++);
        o2 = data.charCodeAt(i++);
        o3 = data.charCodeAt(i++);
        bits = o1<<16 | o2<<8 | o3;
        h1 = bits>>18 & 0x3f;
        h2 = bits>>12 & 0x3f;
        h3 = bits>>6 & 0x3f;
        h4 = bits & 0x3f;
        tmp_arr[ac++] = b64.charAt(h1) + b64.charAt(h2) + b64.charAt(h3) + b64.charAt(h4);
    } while (i < data.length);
    enc = tmp_arr.join('');
    switch (data.length % 3) {
        case 1:
            enc = enc.slice(0, -2) + '==';
        break;
        case 2:
            enc = enc.slice(0, -1) + '=';
        break;
    }
    return enc;
}
function utf8_decode ( str_data ) {
    var tmp_arr = [], i = 0, ac = 0, c1 = 0, c2 = 0, c3 = 0;
    str_data += '';
    while ( i < str_data.length ) {
        c1 = str_data.charCodeAt(i);
        if (c1 < 128) {
            tmp_arr[ac++] = String.fromCharCode(c1);
            i++;
        } else if ((c1 > 191) && (c1 < 224)) {
            c2 = str_data.charCodeAt(i+1);
            tmp_arr[ac++] = String.fromCharCode(((c1 & 31) << 6) | (c2 & 63));
            i += 2;
        } else {
            c2 = str_data.charCodeAt(i+1);
            c3 = str_data.charCodeAt(i+2);
            tmp_arr[ac++] = String.fromCharCode(((c1 & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
            i += 3;
        }
    }
    return tmp_arr.join('');
}

function utf8_encode ( argString ) {
    var string = (argString+''); 
    var utftext = "";
    var start, end;
    var stringl = 0;
    start = end = 0;
    stringl = string.length;
    for (var n = 0; n < stringl; n++) {
        var c1 = string.charCodeAt(n);
        var enc = null;
        if (c1 < 128) {
            end++;
        } else if (c1 > 127 && c1 < 2048) {
            enc = String.fromCharCode((c1 >> 6) | 192) + String.fromCharCode((c1 & 63) | 128);
        } else {
            enc = String.fromCharCode((c1 >> 12) | 224) + String.fromCharCode(((c1 >> 6) & 63) | 128) + String.fromCharCode((c1 & 63) | 128);
        }
        if (enc !== null) {
            if (end > start) {
                utftext += string.substring(start, end);
            }
            utftext += enc;
            start = end = n+1;
        }
    }
    if (end > start) {
        utftext += string.substring(start, string.length);
    }
    return utftext;
}


var counter = 0;
var totalNum = 0;

function downloadPic(url,filePath,fn){
    var r = request(url).pipe(fs.createWriteStream(filePath));
    r.on('close', function(){
        fn();
    }).on('error',function(){
        log('download pic error!');
        if(nextChapterPath && chapterNow < chapterNum){
            downloadChapter(nextChapterPath);
        }else{
            chapterNow++;
        }
    });
}

function downloadFiles(arr,baseDir){
    var length = arr.length;
    var i = 0;
    function loadNext(index){
        downloadPic(arr[i], baseDir + '/' + index + '.jpg',function(){
            i++;
            if(i < length){
                loadNext(i);
            }else{
                console.log('Chapter ' + chapterNow + ' All done!');
                chapterNow++;
                if(nextChapterPath && chapterNow < chapterNum){
                    downloadChapter(nextChapterPath);
                }
            }
        });
    }
    loadNext(i);
}



var baseDir = 'F:/node_test/Pictures/';

var urlReg = new RegExp('qTcms_S_m_murl_e\\s*=\\s*"([\\w+/=]+)"');
var isUrlReg = /^http:\/\//;
var chapterReg = new RegExp('qTcms_S_m_playm\\s*=\\s*"\\W+(\\d+)\\W+"');
var nextChapterReg = new RegExp('qTcms_Pic_nextArr\\s*=\\s*"([/\\w.])html"');

var chapterNow = 0,
    chapterNum = 10;
var nextChapterPath = '';

function log(data){
    fs.appendFile(baseDir + 'log.txt',data + '\r\n','utf8',function(err){
        if(err)
        {
            console.log(err);
        }
    });
}



function downloadChapter(path){
    log('//------------------------------------------------------------------//');

    var page = 'http://some.page.com' + path;
    log('正在读取:' + page);

    var html = '';  
    http.get(page , function(res) {  
        res.setEncoding('utf-8');  
        res.on('data', function(data) {  
            // collect the data chunks to the variable named "html"  
            html += data;  
        }).on('end', function() {

            var dir = baseDir;
            if(chapterReg.test(html)){
                dir +=  '第' + RegExp.$1 + '话';
            }else{
                console.log('找不到章节');
                return;
            }
            if (!fs.existsSync(dir)) {
                fs.mkdirSync(dir);
            }else{
                console.log(dir + ' 已存在!');
                return;
            }
            //
            log('文件存放目录：' + dir);

            var nextChapterReg = new RegExp('qTcms_Pic_nextArr\\s*=\\s*"([/\\w.]+)"');
            if(nextChapterReg.test(html)){
                nextChapterPath = RegExp.$1;
                log('下一个页面地址：' + nextChapterPath);
            }else{
                nextChapterPath = null;
            }

            var url64 = null,
                picArr = null;
            if(urlReg.test(html)){

                url64 = base64_decode (RegExp.$1);

                log('图片数组:\r\n' + url64);

                picArr = url64.split('$');
                picArr =  picArr.filter(function(val){
                    return isUrlReg.test(val);
                });


                downloadFiles(picArr,dir);
            }else{
                //console.log("not found.");
                console.log(html);
            }
        });  
    });  
}

downloadChapter('someurl');