代码如下
import re
import requests
import json
#from multiprocessing import Pool
# 多进程
#url = 'https://maoyan.com/board/4?offset=0'
#r = requests.get(url)
#r.encoding = r.apparent_encoding
#print(r.status_code)
#print(r.request.headers)
# 获取单页
def get_one_page(url):
try:
kv = {'user-agent':'Mozilla/5.0'}
r = requests.get(url,headers=kv)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
print("爬取失败")
# 提取需要的信息
def parse_one_page(html):
pattern = re.compile('.*?board-index.*?>(\d+).*?name">(.*?).*?star">(.*?).*?releasetime">(.*?).*?integer">(.*?).*?fraction">(.*?).*? ', re.S)
items = re.findall(pattern, html)
for item in items:
yield {
'index':item[0],
'title':item[1],
'actor':item[2].strip()[3:],
'time':item[3].strip()[5:],
'score':item[4]+item[5]
}
# 将信息写入文件
def write_to_file(content):
with open('result.txt', 'a', encoding='utf-8') as f:
f.write(json.dumps(content, ensure_ascii=False) + '\n')
f.close()
# 主函数
def main(offset):
url = 'https://maoyan.com/board/4?' + 'offset=' + str(offset)
html = get_one_page(url)
for item in parse_one_page(html):
print(item)
write_to_file(item)
if __name__ == '__main__':
for i in range(10):
main(i*10)
#pool = Pool()
#pool.map(main, [i*10 for i in range(10)])
#多线程 输出结果会乱顺序
运行结果如下:
{“index”: “1”, “title”: “霸王别姬”, “actor”: “张国荣,张丰毅,巩俐”, “time”: “1993-01-01”, “score”: “9.6”}
{“index”: “2”, “title”: “肖申克的救赎”, “actor”: “蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿”, “time”: “1994-10-14(美国)”, “score”: “9.5”}
{“index”: “3”, “title”: “罗马假日”, “actor”: “格利高里·派克,奥黛丽·赫本,埃迪·艾伯特”, “time”: “1953-09-02(美国)”, “score”: “9.1”}
{“index”: “4”, “title”: “这个杀手不太冷”, “actor”: “让·雷诺,加里·奥德曼,娜塔莉·波特曼”, “time”: “1994-09-14(法国)”, “score”: “9.5”}
{“index”: “5”, “title”: “教父”, “actor”: “马龙·白兰度,阿尔·帕西诺,詹姆斯·肯恩”, “time”: “1972-03-24(美国)”, “score”: “9.3”}
{“index”: “6”, “title”: “泰坦尼克号”, “actor”: “莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩”, “time”: “1998-04-03”, “score”: “9.5”}
{“index”: “7”, “title”: “唐伯虎点秋香”, “actor”: “周星驰,巩俐,郑佩佩”, “time”: “1993-07-01(中国香港)”, “score”: “9.2”}
{“index”: “8”, “title”: “千与千寻”, “actor”: “柊瑠美,入野自由,夏木真理”, “time”: “2001-07-20(日本)”, “score”: “9.3”}
{“index”: “9”, “title”: “魂断蓝桥”, “actor”: “费雯·丽,罗伯特·泰勒,露塞尔·沃特森”, “time”: “1940-05-17(美国)”, “score”: “9.2”}
{“index”: “10”, “title”: “乱世佳人”, “actor”: “费雯·丽,克拉克·盖博,奥利维娅·德哈维兰”, “time”: “1939-12-15(美国)”, “score”: “9.1”}
{“index”: “11”, “title”: “天空之城”, “actor”: “寺田农,鹫尾真知子,龟山助清”, “time”: “1992”, “score”: “9.1”}
{“index”: “12”, “title”: “喜剧之王”, “actor”: “周星驰,莫文蔚,张柏芝”, “time”: “1999-02-13(中国香港)”, “score”: “9.2”}
{“index”: “13”, “title”: “辛德勒的名单”, “actor”: “连姆·尼森,拉尔夫·费因斯,本·金斯利”, “time”: “1993-12-15(美国)”, “score”: “9.2”}
{“index”: “14”, “title”: “大闹天宫”, “actor”: “邱岳峰,毕克,富润生”, “time”: “1965-12-31”, “score”: “9.0”}
{“index”: “15”, “title”: “音乐之声”, “actor”: “朱莉·安德鲁斯,克里斯托弗·普卢默,埃琳诺·帕克”, “time”: “1965-03-02(美国)”, “score”: “9.0”}
{“index”: “16”, “title”: “剪刀手爱德华”, “actor”: “约翰尼·德普,薇诺娜·瑞德,黛安·韦斯特”, “time”: “1990-12-06(美国)”, “score”: “8.8”}
{“index”: “17”, “title”: “春光乍泄”, “actor”: “张国荣,梁朝伟,张震”, “time”: “1997-05-30(中国香港)”, “score”: “9.2”}
{“index”: “18”, “title”: “美丽人生”, “actor”: “罗伯托·贝尼尼,尼可莱塔·布拉斯基,乔治·坎塔里尼”, “time”: “1997-12-20(意大利)”, “score”: “9.3”}
{“index”: “19”, “title”: “海上钢琴师”, “actor”: “蒂姆·罗斯,普路特·泰勒·文斯,比尔·努恩”, “time”: “1998-10-28(意大利)”, “score”: “9.2”}
{“index”: “20”, “title”: “黑客帝国”, “actor”: “基努·里维斯,凯瑞-安·莫斯,劳伦斯·菲什伯恩”, “time”: “2000-01-14”, “score”: “9.0”}
{“index”: “21”, “title”: “哈利·波特与魔法石”, “actor”: “丹尼尔·雷德克里夫,鲁伯特·格林特,艾玛·沃特森”, “time”: “2002-01-26”, “score”: “9.1”}
{“index”: “22”, “title”: “加勒比海盗”, “actor”: “约翰尼·德普,凯拉·奈特莉,奥兰多·布鲁姆”, “time”: “2003-11-21”, “score”: “8.9”}
{“index”: “23”, “title”: “指环王3:王者无敌”, “actor”: “伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒”, “time”: “2004-03-15”, “score”: “9.2”}
{“index”: “24”, “title”: “无间道”, “actor”: “刘德华,梁朝伟,黄秋生”, “time”: “2003-09-05”, “score”: “9.1”}
{“index”: “25”, “title”: “射雕英雄传之东成西就”, “actor”: “张国荣,梁朝伟,张学友”, “time”: “1993-02-05(中国香港)”, “score”: “8.9”}
{“index”: “26”, “title”: “楚门的世界”, “actor”: “金·凯瑞,劳拉·琳妮,诺亚·艾默里奇”, “time”: “1998-06-01(美国)”, “score”: “8.9”}
{“index”: “27”, “title”: “蝙蝠侠:黑暗骑士”, “actor”: “克里斯蒂安·贝尔,希斯·莱杰,阿伦·伊克哈特”, “time”: “2008-07-18(美国)”, “score”: “9.3”}
{“index”: “28”, “title”: “教父2”, “actor”: “阿尔·帕西诺,罗伯特·德尼罗,黛安·基顿”, “time”: “1974-12-12(美国)”, “score”: “9.0”}
{“index”: “29”, “title”: “指环王2:双塔奇兵”, “actor”: “伊莱贾·伍德,伊恩·麦克莱恩,丽芙·泰勒”, “time”: “2003-04-25”, “score”: “9.1”}
{“index”: “30”, “title”: “机器人总动员”, “actor”: “本·贝尔特,艾丽莎·奈特,杰夫·格尔林”, “time”: “2008-06-27(美国)”, “score”: “9.3”}
{“index”: “31”, “title”: “天堂电影院”, “actor”: “菲利浦·诺瓦雷,赛尔乔·卡斯特利托,蒂兹亚娜·罗达托”, “time”: “1988-11-17(意大利)”, “score”: “9.2”}
{“index”: “32”, “title”: “活着”, “actor”: “葛优,巩俐,牛犇”, “time”: “1994-05-18(法国)”, “score”: “9.0”}
{“index”: “33”, “title”: “拯救大兵瑞恩”, “actor”: “汤姆·汉克斯,马特·达蒙,汤姆·塞兹摩尔”, “time”: “1998-07-24(美国)”, “score”: “8.9”}
{“index”: “34”, “title”: “哈尔的移动城堡”, “actor”: “倍赏千惠子,木村拓哉,美轮明宏”, “time”: “2004-11-20(日本)”, “score”: “9.0”}
{“index”: “35”, “title”: “阿凡达”, “actor”: “萨姆·沃辛顿,佐伊·索尔达娜,米歇尔·罗德里格兹”, “time”: “2010-01-04”, “score”: “9.1”}
{“index”: “36”, “title”: “盗梦空间”, “actor”: “莱昂纳多·迪卡普里奥,渡边谦,约瑟夫·高登-莱维特”, “time”: “2010-09-01”, “score”: “9.2”}
{“index”: “37”, “title”: “忠犬八公的故事”, “actor”: “Forest,理查·基尔,琼·艾伦”, “time”: “2010-03-12(英国)”, “score”: “9.3”}
{“index”: “38”, “title”: “幽灵公主”, “actor”: “松田洋治,石田百合子,田中裕子”, “time”: “1997-07-12(日本)”, “score”: “8.9”}
{“index”: “39”, “title”: “搏击俱乐部”, “actor”: “爱德华·哈里森·诺顿,布拉德·皮特,海伦娜·伯翰·卡特”, “time”: “1999-10-15(美国)”, “score”: “8.8”}
{“index”: “40”, “title”: “东邪西毒”, “actor”: “张国荣,梁朝伟,刘嘉玲”, “time”: “1994-09-17”, “score”: “8.8”}
{“index”: “41”, “title”: “风之谷”, “actor”: “岛本须美,永井一郎,坂本千夏”, “time”: “1992”, “score”: “8.9”}
{“index”: “42”, “title”: “疯狂原始人”, “actor”: “尼古拉斯·凯奇,艾玛·斯通,瑞安·雷诺兹”, “time”: “2013-04-20”, “score”: “9.5”}
{“index”: “43”, “title”: “当幸福来敲门”, “actor”: “威尔·史密斯,贾登·史密斯,坦迪·牛顿”, “time”: “2008-01-17”, “score”: “8.9”}
{“index”: “44”, “title”: “V字仇杀队”, “actor”: “娜塔莉·波特曼,雨果·维文,斯蒂芬·瑞”, “time”: “2006-03-17(美国)”, “score”: “8.8”}
{“index”: “45”, “title”: “十二怒汉”, “actor”: “亨利·方达,李·科布,马丁·鲍尔萨姆”, “time”: “1957-04-13(美国)”, “score”: “9.1”}
{“index”: “46”, “title”: “放牛班的春天”, “actor”: “热拉尔·朱尼奥,让-巴蒂斯特·莫尼耶,玛丽·布奈尔”, “time”: “2004-10-16”, “score”: “8.8”}
{“index”: “47”, “title”: “三傻大闹宝莱坞”, “actor”: “阿米尔·汗,黄渤,卡琳娜·卡普”, “time”: “2011-12-08”, “score”: “9.1”}
{“index”: “48”, “title”: “勇敢的心”, “actor”: “梅尔·吉布森,苏菲·玛索,帕特里克·麦高汉”, “time”: “1995-05-24(美国)”, “score”: “8.8”}
{“index”: “49”, “title”: “黑客帝国3:矩阵革命”, “actor”: “基努·里维斯,雨果·维文,凯瑞-安·莫斯”, “time”: “2003-11-05”, “score”: “8.8”}
{“index”: “50”, “title”: “速度与激情5”, “actor”: “范·迪塞尔,保罗·沃克,道恩·强森”, “time”: “2011-05-12”, “score”: “9.2”}
{“index”: “51”, “title”: “驯龙高手”, “actor”: “杰伊·巴鲁切尔,杰拉德·巴特勒,亚美莉卡·费雷拉”, “time”: “2010-05-14”, “score”: “9.0”}
{“index”: “52”, “title”: “神偷奶爸”, “actor”: “史蒂夫·卡瑞尔,杰森·席格尔,拉塞尔·布兰德”, “time”: “2010-07-09(美国)”, “score”: “9.0”}
{“index”: “53”, “title”: “少年派的奇幻漂流”, “actor”: “苏拉·沙玛,伊尔凡·可汗,塔布”, “time”: “2012-11-22”, “score”: “9.1”}
{“index”: “54”, “title”: “闻香识女人”, “actor”: “阿尔·帕西诺,克里斯·奥唐纳,加布里埃尔·安瓦尔”, “time”: “1992-12-23(美国)”, “score”: “8.8”}
{“index”: “55”, “title”: “断背山”, “actor”: “希斯·莱杰,杰克·吉伦哈尔,米歇尔·威廉姆斯”, “time”: “2006-01-13(美国)”, “score”: “9.0”}
{“index”: “56”, “title”: “飞屋环游记”, “actor”: “爱德华·阿斯纳,乔丹·长井,鲍勃·彼德森”, “time”: “2009-08-04”, “score”: “8.9”}
{“index”: “57”, “title”: “大话西游之月光宝盒”, “actor”: “周星驰,莫文蔚,吴孟达”, “time”: “2014-10-24”, “score”: “9.6”}
{“index”: “58”, “title”: “飞越疯人院”, “actor”: “杰克·尼科尔森,路易丝·弗莱彻,威尔·萨姆森”, “time”: “1975-11-19(美国)”, “score”: “8.8”}
{“index”: “59”, “title”: “怦然心动”, “actor”: “玛德琳·卡罗尔,卡兰·麦克奥利菲,艾丹·奎因”, “time”: “2010-08-06(美国)”, “score”: “8.9”}
{“index”: “60”, “title”: “美国往事”, “actor”: “罗伯特·德尼罗,詹姆斯·伍兹,伊丽莎白·麦戈文”, “time”: “1984-02-17(美国)”, “score”: “9.1”}
{“index”: “61”, “title”: “致命魔术”, “actor”: “休·杰克曼,克里斯蒂安·贝尔,迈克尔·凯恩”, “time”: “2006-10-20(美国)”, “score”: “8.8”}
{“index”: “62”, “title”: “鬼子来了”, “actor”: “姜文,姜宏波,陈强”, “time”: “2000-05-12(法国戛纳)”, “score”: “8.9”}
{“index”: “63”, “title”: “无敌破坏王”, “actor”: “约翰·C·赖利,萨拉·西尔弗曼,简·林奇”, “time”: “2012-11-06”, “score”: “9.1”}
{“index”: “64”, “title”: “美丽心灵”, “actor”: “罗素·克洛,詹妮弗·康纳利,艾德·哈里斯”, “time”: “2001-12-21(美国)”, “score”: “8.8”}
{“index”: “65”, “title”: “蝙蝠侠:黑暗骑士崛起”, “actor”: “克里斯蒂安·贝尔,迈克尔·凯恩,加里·奥德曼”, “time”: “2012-08-27”, “score”: “8.9”}
{“index”: “66”, “title”: “夜访吸血鬼”, “actor”: “汤姆·克鲁斯,布拉德·皮特,克尔斯滕·邓斯特”, “time”: “1994-11-11(美国)”, “score”: “8.8”}
{“index”: “67”, “title”: “倩女幽魂”, “actor”: “张国荣,王祖贤,午马”, “time”: “2011-04-30”, “score”: “9.2”}
{“index”: “68”, “title”: “哈利·波特与死亡圣器(下)”, “actor”: “丹尼尔·雷德克里夫,鲁伯特·格林特,艾玛·沃特森”, “time”: “2011-08-04”, “score”: “9.0”}
{“index”: “69”, “title”: “本杰明·巴顿奇事”, “actor”: “布拉德·皮特,凯特·布兰切特,塔拉吉·P·汉森”, “time”: “2008-12-25(美国)”, “score”: “8.8”}
{“index”: “70”, “title”: “钢琴家”, “actor”: “艾德里安·布洛迪,艾米莉娅·福克斯,米哈乌·热布罗夫斯基”, “time”: “2002-09-25(法国)”, “score”: “8.8”}
{“index”: “71”, “title”: “触不可及”, “actor”: “弗朗索瓦·克鲁塞,奥玛·希,安娜·勒尼”, “time”: “2011-11-02(法国)”, “score”: “9.1”}
{“index”: “72”, “title”: “熔炉”, “actor”: “孔刘,郑有美,金智英”, “time”: “2011-09-22(韩国)”, “score”: “8.8”}
{“index”: “73”, “title”: “初恋这件小事”, “actor”: “马里奥·毛瑞尔,平采娜·乐维瑟派布恩,阿查拉那·阿瑞亚卫考”, “time”: “2012-06-05”, “score”: “8.8”}
{“index”: “74”, “title”: “大话西游之大圣娶亲”, “actor”: “周星驰,朱茵,莫文蔚”, “time”: “2014-10-24”, “score”: “8.8”}
{“index”: “75”, “title”: “新龙门客栈”, “actor”: “张曼玉,梁家辉,甄子丹”, “time”: “2012-02-24”, “score”: “8.8”}
{“index”: “76”, “title”: “甜蜜蜜”, “actor”: “黎明,张曼玉,曾志伟”, “time”: “2015-02-13”, “score”: “9.2”}
{“index”: “77”, “title”: “素媛”, “actor”: “李来,薛耿求,严志媛”, “time”: “2013-10-02(韩国)”, “score”: “9.1”}
{“index”: “78”, “title”: “小鞋子”, “actor”: “默罕默德·阿米尔·纳吉,Kamal Mirkarimi,Behzad Rafi”, “time”: “1999-01-22(美国)”, “score”: “9.1”}
{“index”: “79”, “title”: “萤火之森”, “actor”: “内山昂辉,佐仓绫音,后藤弘树”, “time”: “2011-09-17(日本)”, “score”: “9.0”}
{“index”: “80”, “title”: “时空恋旅人”, “actor”: “瑞秋·麦克亚当斯,多姆纳尔·格里森,比尔·奈伊”, “time”: “2013-09-04(英国)”, “score”: “8.9”}
{“index”: “81”, “title”: “穿条纹睡衣的男孩”, “actor”: “阿沙·巴特菲尔德,维拉·法梅加,大卫·休里斯”, “time”: “2008-09-12(英国)”, “score”: “9.0”}
{“index”: “82”, “title”: “窃听风暴”, “actor”: “乌尔里希·穆埃,塞巴斯蒂安·科赫,马蒂娜·格德克”, “time”: “2006-03-23(德国)”, “score”: “9.0”}
{“index”: “83”, “title”: “7号房的礼物”, “actor”: “柳承龙,郑镇荣,朴信惠”, “time”: “2013-01-23(韩国)”, “score”: “8.9”}
{“index”: “84”, “title”: “借东西的小人阿莉埃蒂”, “actor”: “志田未来,神木隆之介,大竹忍”, “time”: “2010-07-17(日本)”, “score”: “8.8”}
{“index”: “85”, “title”: “恐怖直播”, “actor”: “河正宇,李璟荣,李大为”, “time”: “2013-07-31(韩国)”, “score”: “8.8”}
{“index”: “86”, “title”: “海豚湾”, “actor”: “里克·奥巴瑞,路易·西霍尤斯,哈迪·琼斯”, “time”: “2009-07-31(美国)”, “score”: “8.9”}
{“index”: “87”, “title”: “忠犬八公物语”, “actor”: “仲代达矢,春川真澄,井川比佐志”, “time”: “1987-08-01(日本)”, “score”: “9.0”}
{“index”: “88”, “title”: “上帝之城”, “actor”: “亚历桑德雷·罗德里格斯,艾莉丝·布拉加,莱安德鲁·菲尔米诺”, “time”: “2002-08-30(巴西)”, “score”: “8.9”}
{“index”: “89”, “title”: “辩护人”, “actor”: “宋康昊,郭度沅,吴达洙”, “time”: “2013-12-18(韩国)”, “score”: “8.8”}
{“index”: “90”, “title”: “七武士”, “actor”: “三船敏郎,志村乔,千秋实”, “time”: “1954-04-26(日本)”, “score”: “9.1”}
{“index”: “91”, “title”: “英雄本色”, “actor”: “狄龙,张国荣,周润发”, “time”: “2017-11-17”, “score”: “9.2”}
{“index”: “92”, “title”: “一一”, “actor”: “吴念真,金燕玲,李凯莉”, “time”: “2000-09-20(法国)”, “score”: “8.9”}
{“index”: “93”, “title”: “完美的世界”, “actor”: “凯文·科斯特纳,克林特·伊斯特伍德,T·J·劳瑟”, “time”: “1993-11-24(美国)”, “score”: “8.9”}
{“index”: “94”, “title”: “海洋”, “actor”: “雅克·贝汉,姜文,兰斯洛特·佩林”, “time”: “2011-08-12”, “score”: “9.0”}
{“index”: “95”, “title”: “爱·回家”, “actor”: “俞承豪,金艺芬,童孝熙”, “time”: “2002-04-05(韩国)”, “score”: “9.0”}
{“index”: “96”, “title”: “黄金三镖客”, “actor”: “克林特·伊斯特伍德,李·范·克里夫,埃里·瓦拉赫”, “time”: “1966-12-23(意大利)”, “score”: “8.9”}
{“index”: “97”, “title”: “我爱你”, “actor”: “宋在浩,李顺才,尹秀晶”, “time”: “2011-02-17(韩国)”, “score”: “9.0”}
{“index”: “98”, “title”: “迁徙的鸟”, “actor”: “雅克·贝汉,Philippe Labro”, “time”: “2001-12-12(法国)”, “score”: “9.1”}
{“index”: “99”, “title”: “阿飞正传”, “actor”: “张国荣,张曼玉,刘德华”, “time”: “2018-06-25”, “score”: “8.8”}
{“index”: “100”, “title”: “龙猫”, “actor”: “帕特·卡洛尔,蒂姆·达利,丽娅·萨隆加”, “time”: “2018-12-14”, “score”: “9.2”}