Python爬虫实现抓取腾讯视频所有电影【实战必学】

 

2019-06-27 23:51:51 阅读数 407  收藏 更多

分类专栏： python爬虫

 

前言
本文的文字及图片来源于网络,仅供学习、交流使用,不具有任何商业用途,版权归原作者所有,如有问题请及时联系我们以作处理。
作者： Python新手学习之家

用python实现的抓取腾讯视频所有电影的爬虫

1.  
   # -*- coding: utf-8 -*-
2.  
   import re
3.  
   import urllib2
4.  
   from bs4 import BeautifulSoup
5.  
   import string, time
6.  
   import pymongo
7.  
    
8.  
   NUM = 0 #全局变量,电影数量
9.  
   m_type = u'' #全局变量,电影类型
10.  
    m_site = u'qq' #全局变量,电影网站
11.  
     
12.  
    #根据指定的URL获取网页内容
13.  
    def gethtml(url):
14.  
    req = urllib2.Request(url)
15.  
    response = urllib2.urlopen(req)
16.  
    html = response.read()
17.  
    return html
18.  
    '''
19.  
    在学习过程中有什么不懂得可以加我的python学习交流扣扣qun，934109170，群里有不错的学习教程与开发工具。
20.  
    '''
21.  
     
22.  
    #从电影分类列表页面获取电影分类
23.  
    def gettags(html):
24.  
    global m_type
25.  
    soup = BeautifulSoup(html) #过滤出分类内容
26.  
    #print soup
27.  
    #
28.  
    tags_all = soup.find_all( 'ul', {'class' : 'clearfix _group' , 'gname' : 'mi_type'})
29.  
    #print len(tags_all), tags_all
30.  
    #print str(tags_all[1]).replace('\n', '')
31.  
     
32.  
    #动作
33.  
    re_tags = r'.+?'
34.  
    p = re.compile(re_tags, re.DOTALL)
35.  
     
36.  
    tags = p.findall(str(tags_all[ 0]))
37.  
    if tags:
38.  
    tags_url = {}
39.  
    #print tags
40.  
    for tag in tags:
41.  
    tag_url = tag[ 0].decode('utf-8')
42.  
    #print tag_url
43.  
    m_type = tag[ 1].decode('utf-8')
44.  
    tags_url[m_type] = tag_url
45.  
     
46.  
    else:
47.  
    print "Not Find"
48.  
    return tags_url
49.  
     
50.  
    #获取每个分类的页数
51.  
    def get_pages(tag_url):
52.  
    tag_html = gethtml(tag_url)
53.  
    #div class="paginator
54.  
    soup = BeautifulSoup(tag_html) #过滤出标记页面的html
55.  
    #print soup
56.  
    #
57.  
    div_page = soup.find_all( 'div', {'class' : 'mod_pagenav', 'id' : 'pager'})
58.  
    #print div_page #len(div_page), div_page[0]
59.  
     
60.  
    #25
61.  
    re_pages = r'(.+?)'
62.  
    p = re.compile(re_pages, re.DOTALL)
63.  
    pages = p.findall(str(div_page[ 0]))
64.  
    #print pages
65.  
    if len(pages) > 1:
66.  
    return pages[-2]
67.  
    else:
68.  
    return 1
69.  
     
70.  
     
71.  
    def getmovielist(html):
72.  
    soup = BeautifulSoup(html)
73.  
     
74.  
    #
75.  
    divs = soup.find_all( 'ul', {'class' : 'mod_list_pic_130'})
76.  
    #print divs
77.  
    for div_html in divs:
78.  
    div_html = str(div_html).replace( '\n', '')
79.  
    #print div_html
80.  
    getmovie(div_html)
81.  
     
82.  
     
83.  
    def getmovie(html):
84.  
    global NUM
85.  
    global m_type
86.  
    global m_site
87.  
     
88.  
    re_movie = r'
89. '

 

p = re.compile(re_movie, re.DOTALL)

 

movies = p.findall(html)

 

if movies:

 

conn = pymongo.Connection( 'localhost', 27017)

 

movie_db = conn.dianying

 

playlinks = movie_db.playlinks

 

#print movies

 

for movie in movies:

 

#print movie

 

NUM += 1

 

print "%s : %d" % ("=" * 70, NUM)

 

values = dict(

 

movie_title = movie[ 1],

 

movie_url = movie[ 0],

 

movie_site = m_site,

 

movie_type = m_type

 

)

 

print values

 

playlinks.insert(values)

 

print "_" * 70

 

NUM += 1

 

print "%s : %d" % ("=" * 70, NUM)

 

 

 

#else:

 

# print "Not Find"

 

 

 

def getmovieinfo(url):

 

html = gethtml(url)

 

soup = BeautifulSoup(html)

 

 

 

#pack pack_album album_cover

 

divs = soup.find_all( 'div', {'class' : 'pack pack_album album_cover'})

 

#print divs[0]

 

 

 

#

 

re_info = r' '

 

p_info = re.compile(re_info, re.DOTALL)

 

m_info = p_info.findall(str(divs[ 0]))

 

if m_info:

 

return m_info

 

else:

 

print "Not find movie info"

 

 

 

return m_info

 

 

 

 

 

def insertdb(movieinfo):

 

global conn

 

movie_db = conn.dianying_at

 

movies = movie_db.movies

 

movies.insert(movieinfo)

 

 

 

if __name__ == "__main__":

 

global conn

 

 

 

tags_url = "http://v.qq.com/list/1_-1_-1_-1_1_0_0_20_0_-1_0.html"

 

#print tags_url

 

tags_html = gethtml(tags_url)

 

#print tags_html

 

tag_urls = gettags(tags_html)

 

#print tag_urls

 

 

 

 

 

for url in tag_urls.items():

 

print str(url[1]).encode('utf-8') #,url[0]

 

maxpage = int(get_pages(str(url[ 1]).encode('utf-8')))

 

print maxpage

 

 

 

for x in range(0, maxpage):

 

#http://v.qq.com/list/1_0_-1_-1_1_0_0_20_0_-1_0.html

 

m_url = str(url[ 1]).replace('0_20_0_-1_0.html', '')

 

movie_url = "%s%d_20_0_-1_0.html" % (m_url, x)

 

print movie_url

 

movie_html = gethtml(movie_url.encode( 'utf-8'))

 

#print movie_html

 

getmovielist(movie_html)

 

time.sleep( 0.1)

 大工告成，以上代码大家都看明白了没？ 如果你看不懂，建议你可以去小编的Python交流.裙 ：一久武其而而流一思（数字的谐音）转换下可以找到了，里面有最新Python教程项目，多练习自然就懂了！