爬取猫眼电影 通过Python异步进行MongoDB存储

讲解：

• 使用Python中PyQuery库 爬去猫眼电影 并存入MongoDB数据库 、txt文档、
• 涉及到Python异步

涉及Python相关库：

from urllib.robotparser import RobotFileParser
import requests
from pyquery import PyQuery
import pymongo
import copy
import asyncio

下面展示一段神奇的代码:

from urllib.robotparser import RobotFileParser
import requests
from pyquery import PyQuery
import pymongo
import copy
import asyncio

offert = 0
offert_tail = 100

async def get_one_page(url):
        rb = RobotFileParser()
        rb.set_url(url)
        rb.read()
        if rb.can_fetch("*", url) == True:
            reponse = requests.get(url)
            if reponse.status_code == 200:
                return reponse.text

async def one_page(html):
    doc = PyQuery(html)
    aa = doc('dd').items()
    item = {}
    for i in aa:
        dd = []
        indexi = i.children('i').text()
        namei = i.find('.name').text()
        start = i.find('.star').text()
        releasetime = i.find('.releasetime').text()
        score = i.find('.score').text()
        item['indexi'] = indexi
        item['namei'] = namei
        item['start'] = start
        item['releasetime'] = releasetime
        item['score'] = score
        write_r(str(item))
        try:
            copyy = copy.deepcopy(item)
            dd.append(copyy)
        except:
            print("添加异常")
        finally:
            mongo(dd)

async def main():
    url = "https://maoyan.com/board/4?offset="
    for i in range(offert, offert_tail, 10):
        urls = url + str(i)
        aa = await get_one_page(urls)
        await one_page(aa)

async def alaws():
    task = loop.create_task(main())
    await asyncio.sleep(0.2)
    try:
        await task
    except asyncio.CancelledError:
        print("取消任务抛出异常")
    finally:
        print("获取结果：",task.result())

def mongo(i):
    conn = pymongo.MongoClient()
    db = conn.dev
    collection = db.dev_test
    try:
        collection.insert_many(i)
    except BaseException as  e:
        print("error:",e)

def write_r(neirong):
    with open('./text.txt','a',encoding='utf-8') as f:
        f.write(neirong + '\n')

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    try:
        print("开启事件")
        loop.run_until_complete(alaws())
    finally:
        print("关闭事件")
        loop.close()

如果关于PyQuery 还不了解的 请看上一章：
https://www.jianshu.com/p/ee588abe500d

联系方式QQ:

294402584