ajax 异步加载页面,爬虫实例—ajax异步（动态）加载的页面信息爬取

ajax_web_parser.py

'''

数据为动态加载(ajax，使用js控制数据)

鼠标滚动(下拉)数据自动显示在页面

通过观察在加载是url的变化，构造出url

url:ttps://knewone.com/discover?page=7

'''

import requests

from bs4 import BeautifulSoup

import time

# 通过url发起请求，获取返回的返回的response信息

def getResponseMsg(url):

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '

'Chrome/68.0.3440.106 Safari/537.36'}

try:

r = requests.get(url, headers=headers)

r.raise_for_status()

r.encoding = r.apparent_encoding

return r.text

except:

return "请求失败"

def parseHtml(html):

soup = BeautifulSoup(html, 'lxml')

# 提取出所在的a标签

imgs = soup.select('a.cover-inner > img')

titles = soup.select('h4.title > a')

links = soup.select('h4.title > a')

for img,title,link in zip(imgs,titles,links):

data = {

'img':img.get('src'),

'title':title.get('title'),

'link':link.get('href')

}

print(data)

#观察发送请求是的url变化，每刷新出内容/discover?page=7&page=4，page+1

def get_more_page(start, end, url): #设置爬取的页面数

for i in range(start, end):

html = getResponseMsg(url+str(i))

parseHtml(html)

time.sleep(2)

if __name__=='__main__':

url = 'https://knewone.com/discover?page=7&page='

#爬取10页数据

get_more_page(1, 10, url)