Python之采集速卖通商品数据

访问网页类：

1.模拟浏览器登陆，获取cookies

2.利用requests访问页面源码

3.cookies具有时效性，如果超出设定时间则重新登陆并更新它

class Browser:
    req = None
    timePre = None

    def __init__(self):
        Browser.timePre = datetime.now()
        if self.get_time():
            self.chrome = webdriver.Chrome()
            self.login("[email protected]", "passwd")
            self.close()
        Browser.req = self.get_request()


    def login(self,username,passwd):
        self.chrome.get("https://login.aliexpress.com/buyer.htm")
        self.chrome.switch_to.frame('alibaba-login-box')
        pwd_btn = self.chrome.find_element_by_name("password")
        act_btn = self.chrome.find_element_by_name("loginId")
        submit_btn = self.chrome.find_element_by_name("submit-btn")
        act_btn.send_keys(username)
        time.sleep(2)
        pwd_btn.send_keys(passwd)
        time.sleep(2)
        submit_btn.send_keys(Keys.ENTER)
        cookies = self.chrome.get_cookies()
        pickle.dump(cookies, open("cookies.pickle", "wb"))

    def get_request(self):
        req = requests.session()
        cookies = pickle.load(open("cookies.pickle", "rb"))
        for cookie in cookies:
            req.cookies.set(cookie['name'], cookie['value'])
        return req

    def get(self,url):
        if self.refresh_req():
            Browser.req = self.get_request()
        html_doc = Browser.req.get(url).text
        return html_doc

    def close(self):
        self.chrome.close()
        self.chrome.quit()

    def refresh_req(self):
        a = Browser.timePre
        b = datetime.now()
        if (int((b-a).seconds)/60) > 10:
            Browser.timePre = b
            return True
        return False

    def get_time(self):
        a = os.path.getmtime("cookies.pickle")
        aa = datetime.fromtimestamp(a)
        b = datetime.now()
        return (int((b-aa).seconds)/60) > 10

连接数据库操作类：https://blog.csdn.net/LINZHENYU1996/article/details/89464037