爬虫实践——selenium、bs4

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.action_chains import ActionChains
import urllib.parse
import urllib.request
from bs4 import BeautifulSoup

一、浏览器的一般设置

对于Chrome：

li_user_agent = ['Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15'
                , 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'
                , 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0'
                , 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1'
                , 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16'
                , 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)'
                ]
chromedriver = 'D:\cqf_软件\python36\chromedriver.exe'   #修改，将Chromedriver放在Python安装路径下的Scripts文件中，因此在启动Chrome时不再指定Chromedriver（写明其存放路径）
chome_options = webdriver.ChromeOptions()
chome_options.add_argument(('--proxy-server=http://' + proxy_ip[0] + ':' + proxy_ip[1])) #添加启动参数（代理IP）
chome_options.add_argument('user-agent='.format(random.choice(li_user_agent))) # 添加报头

chome_options.add_argument('--headless') # 无头浏览器模式
chome_options.add_argument('--disable-gpu')

prefs_images={
   'profile.managed_default_content_settings.images':2}
chome_options.add_experimental_option("prefs",prefs_images) #（不加载图片）
browser = webdriver.Chrome(chromedriver, chrome_options=chome_options)

对于PhantomJS：

dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (random.choice(li_user_agent)) #设置浏览器报头
dcap["phantomjs.page.settings.loadImages"] = False#不加载图片
service_args = ['--proxy=%s' % proxy_ip
                ,'--proxy-type=http'] 设置使用代理IP
browser = webdriver.PhantomJS(desired_capabilities=dcap,service_args=service_args)

二、打开网页并获取网页源码的方式

1、基于requests库

import requests
url='https://book.douban.com/'
headers = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3880.400 QQBrowser/10.8.4554.400 '}

request=requests