Python 爬虫实战案例 : 微信公众号的爬取

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import time
import random
import MySQLdb
import threading
import socket
import math

socket.setdefaulttimeout(60)#这里对整个socket层设置超时时间。后续文件中如果再使用到socket，不必再设置

glock = threading.Lock() #定义全局锁

CATEGORY_URL= ['http://www.we123.com/gzh/onclick/'] #获取地区分类链接
all_url = [] #
ALL_URLS = [] #所有详细页面链接
proxy_list = [] #IP池
URL = 'http://www.we123.com'
PAGE_URL = [] #所有分页链接

#获取Ip池
def get_ip():
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}
    url = 'http://http-webapi.zhimaruanjian.com'#可以使用芝麻代理，好用稳定还不贵
    resp = requests.get(url,headers=headers)
    obj = resp.json() #获取json ip池对象
    for ip in obj:
        arr = 'http://' + str(ip['ip']) + ':' + str(ip['port'])
        proxy_list.append(arr)

#获取页面源码函数
def get_html(url):
    # headers = {}
    user_agent_list  = [
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400',
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'
    ]
    # user_agent = random.choice(user_agent_list)
    headers  = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400'
    }
    # 代理，免费的代理只能维持一会可能就没用了，自行更换
    # proxy_list  = [
    #     "http://27.192.185.62:3252",
    # ]
    # proxy_ip = random.choice(proxy_list)
    # proxies = {'http': proxy_ip}
    # print(str(url))
    try:
        resp = requests.get(url,headers=headers)
        # print("72行：status_code = " + str(resp.status_code))
        # print(type(resp.text))
        # print(resp.url) # 请求的url
        if resp.status_code == 200:
            return resp
        elif resp.status_code == 404:
            return  resp
        elif resp.status_code == 500:
            return  resp
        return  resp
    except RuntimeError:
        print("超时")
        return "error"
    except ConnectionError:
        print("连接超时")
        return "error"
    except RequestException:
        print("http请求父类错误")
        with open('url_exception.txt','a+', encoding='utf-8') as f:
            f.write(str(url))
            f.write('\n')
        return "error"

#获取区域分类链接
def get_categoty_url():
    url = 'http://www.we123.com/gzh/onclick/'
    resp = get_html(url)
    soup = BeautifulSoup(resp.text,'lxml')
    html = soup.select('div.div-subs2 > div.divst-content > div.divst-subs > li > a')
    # 获取区域分类链接
    for i in html:
        city = i['href'].split("/")[-1]
        if (city == '海外' or city == '台湾' or city == '澳门'):
            continue
        url = URL + i['href']
        CATEGORY_URL.append(url)
    print(CATEGORY_URL)


#获取每个区域下所有分页链接
def get_page_url(url):
    city = url.split('/')[-1]
    html = get_html(url)
    if html == "error":
        print("98行：connect url error")
        time.sleep(random.randint(10,20))
        return "error"
    soup = BeautifulSoup(html.text,'lxml')
    #获取总条数
    all_nums = soup.select("div.page > a > b")
    if len(all_nums) == 0:
        return "error"
    else:
        all_nums = soup.select("div.page > a > b")[0].get_text()
    #获取总分页数
    all_pages = math.ceil((int(all_nums) / 30))
    #获取所有分页链接
    all_page_url = []
    for i in range(0,int(all_pages)):
        page_url = 'http://www.we123.com/e/action/ListInfo.php?page=' + str(i) + '&classid=45&line=30&tempid=10&orderby=onclick&myorder=0&totalnum=' + str(all_nums)
        all_page_url.append(page_url)
    return all_page_url

# 获取所有详细页面链接
def get_page_urls():
        global PAGE_URL
        c_url = CATEGORY_URL.pop()
        print('121 行：请求链接' + c_url)
        PAGE_URL = get_page_url(c_url) #获取每个区域下面的所有分页链接

# 获取所有详细页面链接
def get_info_urls():
    while True:
        global  PAGE_URL #设置全局变量
        glock.acquire()  #加锁
        if len(PAGE_URL) == 0:
            glock.release()  #解锁
            print('131 行：CATEGORY_URL 为空')
            break
        else:
            p_url = PAGE_URL.pop()
            print('135 行：请求链接' + p_url)
            glock.release()  #解锁

            glock.acquire()  #加锁
            html = get_html(p_url)
            if html == "error":
                print("141行：connect url error")
                time.sleep(2)
                return
            soup = BeautifulSoup(html.text,'lxml')
            info_urls = soup.select('div.gzhRight > div.gzh_list > ul > li > a')
            for x in info_urls:
                i_url = URL + x['href']
                ALL_URLS.append(i_url)
            print("库存链接共:" + str(len(ALL_URLS)))
        glock.release()  #解锁
#获取每一页需要的数据
def get_data():
    while True:
        global ALL_URLS  #设置全局变量
        glock.acquire()  #加锁
        print("当前库存："+str(len(ALL_URLS)))
        if len(ALL_URLS) == 0:
            glock.release()  #解锁
            print('159 行 ：ALL_URLS 为空')
            break
        else:
            url = ALL_URLS.pop()
            print("开始抓取数据：" + url)
            glock.release() #解锁
            time.sleep(1) #睡眠1秒钟
            html = get_html(url)
            if html == "error":
                print("168行：connect url error")
                time.sleep(random.randint(2, 4))
                return
            html.encoding='utf-8' #显式地指定网页编码，一般情况可以不用
            soup = BeautifulSoup(html.text,'lxml')
            #公众号名称
            names = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > h1')
            #微信号id
            accounts = []
            accounts.append(soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > p')[0])
            #微信头像
            imgs = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > img')
            #公众号二维码
            QR_codes= soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_right >  img')
            #介绍
            descs = soup.select('div.artcleLeft > div.xcxnry > div.xcxinfo')
            #公众号分类
            categorys = []
            category = ''
            cate = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.xcx_p > span > a')
            if not len(cate) == 0:
                category = cate[0].get_text()
            else:
                category = '综合'
            glock.acquire()  #加锁
            for name,account,img,QR_code,desc in zip(names,accounts,imgs,QR_codes,descs):
                data = {
                    'name':name.get_text(),
                    'category':category,
                    'account':account.get_text().split("：")[-1],
                    'img':img['src'],
                    'QR_code':QR_code['src'],
                    'desc':desc.get_text()
                }
                add_data(data,url)
            glock.release() #解锁
#添加数据
def add_data(data,url):
    con = MySQLdb.connect('127.0.0.1','root','root','test',charset="utf8",use_unicode=True)
    cursor = con.cursor()
    # exit()
    insert_sql = """
        insert ignore into weixin5(w_name,category,account,img,QR_code,introduce)
        VALUES (%s,%s,%s,%s,%s,%s)
        """
    print('212行 ：' + data['name'] + '_' + data['account'] + '添加成功！-' + url)
    try:
        cursor.execute(insert_sql,(data['name'],data['category'],data['account'],data['img'],data['QR_code'],str(data['desc'])))
        con.commit()
    except:
        ALL_URLS.insert(0,url)
        print("218行：" + URL + '插入失败')
        con.rollback()
    con.close()

# 将时间字符串转化为时间戳
def time_to(dt):
    timeArray = time.strptime(dt, "%Y年%m月%d日")
    timestamp = int(time.mktime(timeArray))
    return timestamp

#启动多线程爬取
def main():
    for x in range(3):
        th = threading.Thread(target=get_info_urls)
        th.start()
         # get_info_urls()
    time.sleep(3)
    for x in range(5):
        th = threading.Thread(target=get_data)
        th.start()

if __name__ == '__main__':
    # 计时
    t1 = time.time()
    # 调用函数
    get_ip() #获取ip池
    get_page_urls()
    time.sleep(2)
    # get_categoty_url()
    main()
    print(time.time() - t1)