用3种方式爬取动态网站数据：华南粮网

需求：爬取 华南粮网：https://gdgrain.com/#!/list?params=%7B%22type%22:7%7D 的标题和时间。

该网站查看网页源代码发现，无标题数据：
该网站通过requests请求是请求不到数据的，可以通过下面的3种方法来获取数据。

1. 分析网站，找出post请求

网站分析如图：

代码如下：

import requests
import json
import time

url = 'https://gdgrain.com/sgtcPortals-front/sgtc/portals/SPql001'
headers = {
    # 'Accept': 'application/json, text/plain, */*',
    # 'Accept-Encoding': 'gzip, deflate, br',
    # 'Accept-Language': 'zh-CN,zh;q=0.9',
    # 'Connection': 'keep-alive',
    # 'Content-Length': '59',
    'Content-Type': 'application/json;charset=UTF-8',
    # 'Host': 'gdgrain.com',
    # 'Origin': 'https://gdgrain.com',
    # 'Referer': 'https://gdgrain.com/',
    # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',
    # 'X-Requested-With': 'XMLHttpRequest'
}
form_data = {"pageNo": 1, "pageSize": 10, "iColumnId": 5, "channelCode": "04"}  # 字典类型，需要转成json类型
resp = requests.post(url=url, data=json.dumps(form_data), headers=headers)
json_data = json.loads(resp.text)
for data in json_data['result']['orderBeanLists']:
    title = data['sTitle']
    publish_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(data['dCreateDate']) / 1000))
    print(title, publish_time)

执行结果如下：

2. 用selenium

# -*- encoding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.chrome.options import Options


def selenium_test(url):
    # 设置无头浏览器，字符编码，请求头等信息，防止反爬虫检测
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('lang=zh_CN.UTF-8')
    UserAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'
    chrome_options.add_argument('User-Agent=' + UserAgent)
    browser = webdriver.Chrome(chrome_options=chrome_options)
    browser.get(url)

    data_list = browser.find_elements_by_xpath('//ul[@class="listUl ng-scope"]/li')
    for data in data_list:
        title = data.find_element_by_xpath('./p').text
        publish_time = data.find_element_by_xpath('./em').text
        print(title, publish_time)


selenium_test(url='https://gdgrain.com/#!/list?params=%7B%22type%22:5%7D')

执行结果如下：

3. 用asyncio + pyppeteer来爬取

用selenium速度慢，对版本配置要求严苛，并且不支持异步。而pyppeteer支持异步，并且效率相对 Selenium 来说也提⾼了不少。

pyppeteer的环境搭建可参考博客：pyppeteer的环境搭建，常见参数及2个案例

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import asyncio
from pyppeteer import launch
from scrapy import Selector


async def main():
    # 浏览器 启动参数
    start_parm = {
        # 启动chrome的路径
        "executablePath": r"E:\tmp\chrome-win\chrome.exe",
        # 关闭无头浏览器 默认是无头启动的
        "headless": False,
    }
    # 创建浏览器对象，可以传入 字典形式参数
    browser = await launch(**start_parm)
    # 创建一个页面对象， 页面操作在该对象上执行
    page = await browser.newPage()

    await page.goto('https://gdgrain.com/#!/list?params=%7B%22type%22:5%7D')
    page_text = await page.content()  # 页面内容
    selecter = Selector(text=page_text)
    data_list = selecter.xpath('//ul[@class="listUl ng-scope"]/li')
    for data in data_list:
        title = data.xpath('./p/text()').extract_first()
        publish_time = data.xpath('./em/text()').extract_first()
        print(title, publish_time)
    await browser.close()  # 关闭浏览器对象


asyncio.get_event_loop().run_until_complete(main())   # 创建异步池并执行main函数。

把上面代码优化之后：

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import asyncio
from pyppeteer import launch
from scrapy import Selector


def parse_content(page_text):
    selecter = Selector(text=page_text)
    data_list = selecter.xpath('//ul[@class="listUl ng-scope"]/li')
    for data in data_list:
        title = data.xpath('./p/text()').extract_first()
        publish_time = data.xpath('./em/text()').extract_first()
        print(title, publish_time)


async def main():
    # 浏览器 启动参数
    start_parm = {
        # 启动chrome的路径
        "executablePath": r"E:\tmp\chrome-win\chrome.exe",
        # 关闭无头浏览器 默认是无头启动的
        "headless": False,
    }
    # 创建浏览器对象，可以传入 字典形式参数
    browser = await launch(**start_parm)
    # 创建一个页面对象， 页面操作在该对象上执行
    page = await browser.newPage()
    await page.goto('https://gdgrain.com/#!/list?params=%7B%22type%22:5%7D')
    page_text = await page.content()  # 页面内容
    await browser.close()  # 关闭浏览器对象
    return page_text


if __name__ == '__main__':
    text = asyncio.get_event_loop().run_until_complete(main())   # 创建异步池并执行main函数
    parse_content(page_text=text)

执行结果如下图：