Python3爬虫实例-爬取双色球开奖记录并保存到数据库

import requests
from fake_useragent import UserAgent
from lxml import etree
import pymysql


class SqlHelper(object):

    def __init__(self):
        self.connect()

    def connect(self):
        self.conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='', db='shuangseqiu', charset='utf8')
        self.cursor = self.conn.cursor(cursor=pymysql.cursors.DictCursor)

    def get_one(self, sql, args):
        self.cursor.execute(sql, args)
        result = self.cursor.fetchone()
        return result

    def modify(self, sql, args):
        self.cursor.execute(sql, args)
        self.conn.commit()

    def close(self):
        self.cursor.close()
        self.conn.close()

headers={
    'User-Agent':UserAgent().chrome
}
url='https://datachart.500.com/ssq/?expect=50'


response=requests.get(url,headers=headers)
e=etree.HTML(response.text)
nos=e.xpath('//tbody[@id="tdata"]/tr/td[1]/text()')
trs=e.xpath('//tbody[@id="tdata"]/tr[not(@class)]')

client=SqlHelper()

is_exist_sql='select * from ssq where no=%s'
insert_sql='insert into ssq value(0,%s,%s,%s)'

record=0
nos.reverse()
for no in nos:
    result=client.get_one(is_exist_sql,[no])
    if result!=None:
        break
    record+=1
print('要执行的任务条数:',record)

trs.reverse()
for i in range(record):
    rb='-'.join(trs[i].xpath('./td[@class="chartBall01"]/text()'))
    bb = trs[i].xpath('./td[@class="chartBall02"]/text()')[0]
    print('插入数据:',nos[i], ':', rb, bb)
    client.modify(insert_sql,[nos[i],rb,bb])

client.close()
# for no,tr in zip(nos,trs):
#     rb='-'.join(tr.xpath('./td[@class="chartBall01"]/text()'))
#     bb=tr.xpath('./td[@class="chartBall02"]/text()')[0]
#     print(no,':',rb,bb)