Python使用Selenium + PhantomJS抓取动态网页：今日头条

下载安装PhantomJS，这是一个无界浏览器，不能使用pip安装，以及安装lxml库，Selenium库。另外BeautifulSoup库和Selenium自带的css选择器也想用用看。

#coding:utf8
#python27
import re
from selenium import webdriver
import time
from lxml.html import fromstring

class Toutiao():
    links = []

首先是获取今日头条指定栏目下的新闻链接

class Toutiao():
    #省略...
    def getLinks(self,url):
        #调试时可以使用火狐浏览器，需事先下载火狐的驱动
        #driver = webdriver.Firefox(executable_path=r'C:\Python27\geckodriver.exe')
        driver = webdriver.PhantomJS()  
        driver.get(url)
        #设置隐式等待，最多等待10s
        driver.implicitly_wait(5) 
        for i in range(2):
            #设置下拉次数模拟下拉滚动条加载网页
            driver.execute_script("window.scrollBy(0,700)")
            driver.implicitly_wait(2)
        #将网页源码保存
        html = driver.page_source
        #一定要关闭
        driver.close()
        tree = fromstring(html)

另外，在爬取的时候，发现今日头条网站页面的元素属性并不是固定的，而且链接也不总是http,有时候是https，因此需要两者都考虑到。

    def getLinks(self,url):
        #省略...
        try:
            list = tree.xpath('//a[@class="link"]/@href')
        except:
            list = tree.xpath('//a[@class="link title"]/@href')
        for i in list:
            #需将抓到的链接拼接完全
            link = 'http://www.toutiao.com' + i
            print 'Get link:',link
            self.links.append(link)

看一下抓到的链接：

if __name__=='__main__':
    Toutiao().getLinks('http://www.toutiao.com/')

接下来自然是抓取新闻页面的内容了。我们只需要今日头条自己的新闻，并且是本日的新闻，今日头条里面有一部分是视频新闻，还有一些是其他网站的新闻，因此我们需要做一个过滤。

时间上的过滤方法：将网页中的发布时间抓取下来与本地时间做一个对比

def isToday(fb_time):
    today = time.strftime('%Y-%m-%d',time.localtime(time.time()))
    fb_time = re.findall('(.*?) .*?',fb_time)[0]
    if fb_time != today:
        return True
    else:
        return False

接下来抓取正文，这里用下Selenium自带的选择器：

def getNews(self,url):
        print url,'开始抓取'
        driver = webdriver.PhantomJS()
        #driver = webdriver.Firefox(executable_path=r'C:\Python27\geckodriver.exe') 
        driver.get(url)
        driver.implicitly_wait(5) 
        #文章网址
        wen_zhang_url = url
        #文章标题,遇到视频跳出
        try:
            title = driver.find_element_by_xpath('//h1[@class="article-title"]').text
        except:
            print url,'条件不符，跳过'
            driver.close()
            exit(0)
        #发布时间
        try:
            fa_bu_shi_jian = driver.find_elements_by_xpath('//div[@class="articleInfo"]/span')[2].text
        except:
            try:
                fa_bu_shi_jian = driver.find_elements_by_xpath('//div[@class="article-sub"]/span')[2].text
            except:
                try:
                    fa_bu_shi_jian = driver.find_elements_by_xpath('//div[@class="article-sub"]/span')[1].text
                except:
                    fa_bu_shi_jian = driver.find_elements_by_xpath('//div[@class="articleInfo"]/span')[1].text
        #评论数量
        if self.isToday(fa_bu_shi_jian):
            print url,'日期不符，跳过'
            exit(0)
        try:
            submit_button = driver.find_element_by_class_name('c-load-more')
            submit_button.click()
        except:
            pass
        time.sleep(3)
        page = driver.page_source
        pin_lun_shu_liang = driver.find_element_by_xpath('//a[@class="share-count"]//span').text
        if int(pin_lun_shu_liang) :
            try:
                self.getCommen(url,page)
            except:
                pass
        else:
            print url,'无评论'
        #文章来源
        try:
            wen_zhang_source = driver.find_elements_by_xpath('//div[@class="articleInfo"]/span')[0].text
        except:
            wen_zhang_source = driver.find_elements_by_xpath('//div[@class="article-sub"]/span')[0].text
        #文章正文
        wen_zhang_zheng_wen = ''
        wen_zhang = driver.find_elements_by_xpath('//div[@class="article-content"]//p')
        for i in wen_zhang:
            wen_zhang_zheng_wen += i.text
        #抓取时间
        #now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
        now = time.time()
        #抓取网站
        zhan_dian = u'今日头条'
        #图片链接
        imgs = ''
        try:
            img = driver.find_elements_by_xpath('//div[@class="article-content"]//p/img')
            for i in img:
                i = i.get_attribute('src')
                imgs = imgs + i + ' '
        except:
            imgs = ''
        #文章栏目
        wen_zhang_lan_mu = wen_zhang = driver.find_element_by_xpath('//a[@ga_event="click_channel"]').text
        #文章作者
        author = None
        #关键词
        guan_jian_ci = []
        try:
            cis = driver.find_elements_by_xpath('//a[@class="label-link"]')
        except:
            cis = driver.find_elements_by_xpath('//a[@class="tag-item"]')
        for i in cis:
            guan_jian_ci.append(i.text)
        #阅读数量
        readers = None
        #主键     
        driver.close()
        data = {'wen_zhang_wang_zhi':wen_zhang_url,\
                'wen_zhang_biao_ti':title,\
                'fa_bu_shi_jian':fa_bu_shi_jian,\
                'ping_lun_shu_liang':pin_lun_shu_liang,\
                'wen_zhang_lai_yuan':wen_zhang_source,\
                'wen_zhang_zheng_wen':wen_zhang_zheng_wen,\
                'do_time':now,\
                'zhan_dian':zhan_dian,\
                'tu_pian_lian_jie':imgs,\
                'wen_zhang_lan_mu':wen_zhang_lan_mu,\
                'wen_zhang_zuo_zhe':author,\
                'xiang_guan_biao_qian':guan_jian_ci,\
                'guan_jian_ci':None,\
                'yue_du_shu':readers,\
                '_id':wen_zhang_url}

需要存到数据库，团队用的mongodb。这里我另写到一个文件

#coding:utf8
#clientDb.py
import pymongo

client = pymongo.MongoClient('你自己的ip',端口)
db = client.news

def InsertNews(data):
    collection = db.content
    collection.insert(data)

def InsertComments(data):
    collection = db.comment
    collection.insert(data)

接着getNews:

from clientDb import InsertNews,InsertComments
#省略...
class Toutiao():
    #省略...
    def getNews(self,url):
        #省略...
        #存到数据库
        try:
            InsertNews(data)
        except:
            print url,'该文章已存在'
            exit(0)
        print url,'文章获取成功'

还有需要获取评论，这里我们试着用下BeautifulSoup库:

class Toutiao():
    #.....
    def getCommen(self,url,page):
        soup = BeautifulSoup(page,'html.parser')
        all = soup.find_all('div',class_='c-content')
        comments = []
        for i in all:
            id = 'http://www.toutiao.com'+i.find('a').get('href')
            name = i.find('a').get_text()
            c_content =  i.find('p').get_text()
            c_time = i.find('span',class_='c-create-time').get_text()
            if c_time[1] == u'\u5206':
                c_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            else:
                now = time.strftime('%H',time.localtime(time.time()))
                time1 = int(now) - int(c_time[0])
                if time1<10:
                    time1 = '0'+str(time1)+':'
                else:
                    time1 = str(time1)+':'
                now = now+':'
                c_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
                c_time = c_time.replace(now,time1)
            try:
                r_count =  i.find('span',class_='c-reply-count').get_text()
            except:
                r_count = None
            z_count = i.find('span',title='点赞').get_text()
            #z_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
            z_time = time.time()
            c_url = u'今日头条'
            comment = {}
            #comment['news_url'] = url
            comment['ping_lun_id'] = id
            comment['yong_hu_ming'] = name
            comment['xing_bie'] = None
            comment['yong_hu_deng_ji'] = None
            comment['yong_hu_sheng_fen'] = None
            comment['ping_lun_nei_rong'] = c_content
            comment['hui_fu_shu'] = r_count
            comment['ping_lun_shi_jian'] = c_time
            #print comment['ping_lun_shi_jian']
            comment['do_time'] = z_time
            comment['dian_zan_shu'] = z_count
            comment['zhan_dian'] = c_url
            comment['_id'] = id+url
            comments.append(comment)
        try:
            InsertComments(comments)
        except:
            print url,'该评论已存在'
            exit(0)
        print url,'评论获取成功'

到这里就算基本完了，现在试试最终效果：

if __name__=='__main__':
    test = Toutiao()
    test.getLinks('http://www.toutiao.com/')
    count = len(test.links)
    print '共%d' % count
    while len(test.links):
        url = test.links.pop()
        try:
            test.getNews(url)
        except:
            pass
        count -= 1
        print '余%d' % count

效果大概就是这样：

数据库中效果：

源码在这里源码