这个技术有空再学习一下，就先收藏在这里了。
转载自：http://www.cnblogs.com/ry123/p/4734665.html
纯属初学...有很多需要改进的地方,请多多指点...
目标是抓取58同城 这个大分类下的列表数据: http://cd.58.com/caishui/?PGTID=14397169455980.9244072034489363&ClickID=1
简单分析:
1. 按照以下二级分类来获取每个列表的数据,
2. 主要分页: 可以看出,其分页是pn5 这里设置的,那么这个5就是页码了.
http://cd.58.com/dailijizh/pn5/?PGTID=117742907188706554997826849&ClickID=1
3. 电话号码: 实在隐藏的div里面,点击 联系商家即可看到.但是对于程序来说.只可以直接取得的.
代码如下:
//抓取58数据var http = require("http"),
cheerio = require("cheerio"),
mongoose = require('mongoose');
db = mongoose.createConnection('mongodb://127.0.0.1:27017/crawl58');
db.on('error', function (error) {
console.log('mongodb连接错误: ' + error);
});//存储数据var mongooseSchema = new mongoose.Schema({
url: {type: String},//抓取地址
type: {type: String},//类型
content: {type: String},//抓取地址
updateTime: {type: Date, default: Date.now},//数据抓取时间
flag: {type: String, default: 0} //用于判断是否抓取过 0表示详情没有抓取过.});// modelvar mongooseModel = db.model('pageList', mongooseSchema);//代理记账//抓取数据var proxy = [ //代理
{ip: '120.203.159.14', port: '8118'},
{ip: '111.161.246.233', port: '8118'},
{ip: '58.30.233.196', port: '8118'},
{ip: '113.215.0.130', port: '80'},
{ip: '183.218.63.179', port: '8181'},
{ip: '120.198.245.36', port: '8080'},
{ip: '120.203.158.149', port: '8118'},
{ip: '124.240.187.89', port: '80'},
{ip: '218.204.140.105', port: '8118'},
{ip: '175.1.79.63', port: '80'}
];var proxyIndex = 5;var flag = false;//判断是否为最后一页var pageNo = 1;function crawl() {
console.log('正在抓取 页码: ' + pageNo); //url需要手动设置一下,每个分类抓完毕后,切换到下一个分类
//var url = 'http://cd.58.com/dailijizh/pn' + pageNo + '/?PGTID=1007041601886955933022299' + pageNo + '&ClickID=1';
var url = 'http://cd.58.com/nashuishenbao/pn' + pageNo + '/?PGTID=1007041601886955933022299' + pageNo + '&ClickID=1'; var type='纳税申报';//这里需要手动设置一下分类,对应url分类
if (flag) {
console.log('抓取完毕.总页数为:' + pageNo); return false;
} var option = {
host: proxy[proxyIndex].ip,
port: proxy[proxyIndex].port,
method: 'GET',//这里是发送的方法 path: url,
header: { 'Host': 'cd.58.com', 'Connection': 'keep-alive', 'Cache-Control': 'max-age=0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,p_w_picpath/webp,*/*;q=0.8', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 'Referer': url, 'Accept-Encoding': 'gzip, deflate, sdch', 'Accept-Language': 'zh-CN,zh;q=0.8', 'Cookie': 'userid360_xml=6B337B22E8098342C5F725D4F58495C6; time_create=1442050990222; id58=05dzXVSaeAcJgzn9Crp9Ag==; bdshare_firstime=1419409592050; bangbangguoqi=true; ppqp=1; tj_ershoubiz=true; tj_ershounobiz=true; CNZZDATA30017898=cnzz_eid%3D443859762-1419406677-%26ntime%3D1431055823; ag_fid=WeYSRnDPQwUjsUJF; myfeet_tooltip=end; quanmyy=forfirst; __ag_cm_=1439442804516; bangbigtip2=1; nearby=NOTSHOW; ipcity=cd%7C%u6210%u90FD; sessionid=4019a46c-3b78-45f9-8af1-d5d576171b60; 58home=cd; bangbangid=1080863912864997567; cookieuid1=05dvUVXOs3ZTEwlzHrnMAg==; __autma=253535702.1952421463.1439442813.1439598477.1439610035.5; __autmc=253535702; __autmz=253535702.1439610035.5.2.autmcsr=cd.58.com|autmccn=(referral)|autmcmd=referral|autmcct=/caishui/; final_history=19947936375429%2C20303113064713%2C16884696076038%2C18742095746434%2C22669284355361; ag_fid=WeYSRnDPQwUjsUJF; __utmt_pageTracker=1; city=cd; Hm_lvt_3bb04d7a4ca3846dcc66a99c3e861511=1439452109,1439458833,1439516942,1439598477; Hm_lpvt_3bb04d7a4ca3846dcc66a99c3e861511=1439627751; __utma=253535702.1249887847.1419409519.1439618478.1439625451.38; __utmb=253535702.20.10.1439625451; __utmc=253535702; __utmz=253535702.1439625451.38.15.utmcsr=cd.58.com|utmccn=(referral)|utmcmd=referral|utmcct=/dailijizh/pn2/; new_session=0; init_refer=http%253A%252F%252Fcd.58.com%252Fdailijizh%252Fpn2%252F%253FPGTID%253D198304873188692623092226919; new_uv=41'
}
}; //http.request(option, function (res) {//这里为使用代理IP,还有bug,暂时没有解决掉.
http.get(url, function (res) { var data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on("end", function () { //解析数据并存入数据库
var $ = cheerio.load(data); if ($('a.next', 'div.pager').length < 1) {
flag = true;//设置 抓取完毕的标志 } var item = {
url: url,
type: type,
content: data
} //保存列表数据
mongooseModel.create(item, function (error) { if (error) {
console.log(error);
} else {
console.log('保存成功 页码: ' + pageNo + ' ' + url); if (proxyIndex = 10) {
proxyIndex = 0;
} else {
proxyIndex = proxyIndex + 1;
}
pageNo = pageNo + 1;
setTimeout(crawl, 5020);//设置为5秒以上,就不会被跳转验证页面, O(∩_∩)O哈哈~被我发现了. 其实最好是使用代理. }
});
});
}).on("error", function (error) {
console.log('抓取错误: ' + error.message);
});
};//开始抓取数据crawl();