数据采集之豆瓣读书

爬取【豆瓣读书https://book.douban.com/latest?icn=index-latestbook-all】，并将爬取的内容添加导数据库中。

数据库

cmd
C:\Users\admin>mysql -u root -p
mysql> show databases;
mysql> use spier;
mysql> create table douban(Id Int,BookName varchar(100),Score varchar(20),Autor varchar(100),Press varchar(200),Pubdate varchar(20),describ varchar(3000))ENGINE=InnoDB DEFAULT CHARSET=utf8;  //建表

运行代码

from lxml import etree
import requests
import pymysql

def get_html(url,headers,time=10):  #get请求通用函数，去掉了user-agent简化代码
    try:
        r = requests.get(url, headers=headers,timeout=time)  # 发送请求
        r.encoding = r.apparent_encoding  # 设置返回内容的字符集编码
        r.raise_for_status()  # 返回的状态码不等于200抛出异常
        return r.text  # 返回网页的文本内容
    except Exception as error:
      print(error)

bookinfos=[]
def parse_html(html):
    doc=etree.HTML(html)
    for row in doc.xpath("//div[@id='content']//li"):
        bookname=row.xpath("./div/h2/a/text()")[0]
        score=row.xpath("./div/p[1]/span[2]/text()")[0].strip()
        apt=row.xpath("./div/p[2]/text()")[0].strip()
        aptt=apt.split('/')
        author=aptt[0]
        press=aptt[1]
        pubdate=aptt[2]
        describe=row.xpath("./div/p[3]/text()")[0].strip()
        bookinfo=(bookname,score,author,press,pubdate,describe)
        bookinfos.append(bookinfo)

def save_mysql(sql,vals,**dbinfo):
    try:
        conn = pymysql.Connect(**dbinfo)
        cursor=conn.cursor()
        cursor.executemany(sql,vals)#批量处理添加数据
#         for book in vals:
#             cursor.execute(sql,book)
        conn.commit()
        print("insert success")
    except:
        conn.rollback()
        print("insert faild")
    finally:
        cursor.close()
        conn.close()
if __name__=="__main__":#main函数，流程设计
    url='https://book.douban.com/latest?icn=index-latestbook-all'
    headers={
    "User-Agent":
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64)\
    AppleWebKit/537.36 (KHTML, like Gecko) \
    Chrome/89.0.4389.82 Safari/537.36"
}
    html=get_html(url,headers)
    parse_html(html)
    print(bookinfos)
    #数据库要改为自己的内容
    parms={   
         "host":'127.0.0.1',
        "port":3306,
        "user":'root',
        "passwd":'123456',
        "db":'spier',
        "charset":'utf8'
    }
    sql="insert into douban(BookName,Score,Autor,Press,Pubdate,describ) values(%s,%s,%s,%s,%s,%s)"
    save_mysql(sql,bookinfos,**parms)