python爬虫 爬取斗鱼直播数据

from time import sleep

import requests
from bs4 import BeautifulSoup

# 爬取分类页面数据

#获取斗鱼分类页面数据
def get_directory():
    #获取网页
    url = 'https://www.douyu.com/directory'
    html = requests.get(url).text
    # print(html)
    #解析网页
    soup = BeautifulSoup(html, 'lxml')
    unit_list = soup.find_all('li', class_='unit')
    # print(unit_list)
    #抓取想要的数据
    results = []
    for unit in unit_list:
        href = 'https://www.douyu.com' + unit.find('a')['href']
        # print(href)
        images = unit.find('img')['data-original']
        # print(images)
        title = unit.find('p').text
        # print(title)
        #生成字典
        dict1 = {
            'href': href,
            'image': images,
            'title': title
        }
        sleep(2)
        results.append(dict1)
    return results


def main():
    results = get_directory()
    for result in results:
        href = result['href']
        directory_image = result['image']
        directory_title = result['title']

        detail_list = get_zhibo(href, directory_title, directory_image)


if __name__ == '__main__':
    main()
#获取直播页面信息
def get_zhibo(href, directory_title, directory_image):
        # 自定义请求头   
        headers = {
            'Host': 'www.douyu.com',
            'Referer': 'https://www.douyu.com/directory',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) '
                           'AppleWebKit/537.36 (KHTML, like Gecko) '
                           'Chrome/63.0.3239.132 Safari/537.36'
        }
        # 拿到网页
        html = requests.get(href, headers=headers).text
        # print(html)
        # 解析网页
        soup = BeautifulSoup(html, 'lxml')
        contentbox = soup.find('ul', id='live-list-contentbox')
        li_list = contentbox.find_all('li')
        detail_list = []
        # 提取有用信息
        for li in li_list:
            title = li.find('a')['title']
            # print(title)
            image = li.find('img')['data-original']
            # print(image)
            classify = li.find('span', class_='tag').text
            # print(classify)
            name = li.find('span', class_='dy-name').text
            # print(name)
            num = li.find('span', class_='dy-num').text
            # print(num)
            # 生成字典
            dict2 = {'directory_title': directory_title,
                     'directory_image': directory_image,
                     'detail': {
                         'title': title,
                         'image': image,
                         'classify': classify,
                         'name': name,
                         'num': num
                     }
                }
            sleep(2)
            detail_list.append(dict2)
        return detail_list
# 保存内容
def save_content(detail_list, directory_title):
    filename = directory_title + '.txt'
    for x in detail_list:
        with open(filename, 'a') as f:
            f.write(x)

#最后代码整理如下

from time import sleep

import requests
from bs4 import BeautifulSoup


def get_directory():
    url = 'https://www.douyu.com/directory'
    html = requests.get(url).text
    soup = BeautifulSoup(html, 'lxml')
    unit_list = soup.find_all('li', class_='unit')
    results = []
    for unit in unit_list:
        href = 'https://www.douyu.com' + unit.find('a')['href']
        images = unit.find('img')['data-original']
        title = unit.find('p').text
        dict1 = {
            'href': href,
            'image': images,
            'title': title
        }
        sleep(2)
        results.append(dict1)
    return results


def get_zhibo(href, directory_title, directory_image):
    headers = {
        'Host': 'www.douyu.com',
        'Referer': 'https://www.douyu.com/directory',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/63.0.3239.132 Safari/537.36'
    }
    html = requests.get(href, headers=headers).text
    soup = BeautifulSoup(html, 'lxml')
    contentbox = soup.find('ul', id='live-list-contentbox')
    li_list = contentbox.find_all('li')
    detail_list = []
    for li in li_list:
        title = li.find('a')['title']
        image = li.find('img')['data-original']
        classify = li.find('span', class_='tag').text
        name = li.find('span', class_='dy-name').text
        num = li.find('span', class_='dy-num').text
        dict2 = {'directory_title': directory_title,
                 'directory_image': directory_image,
                 'detail': {
                     'title': title,
                     'image': image,
                     'classify': classify,
                     'name': name,
                     'num': num
                 }
            }
        sleep(2)
        detail_list.append(dict2)
    return detail_list


def save_content(detail_list, directory_title):
    filename = directory_title + '.txt'
    for x in detail_list:
        with open(filename, 'a') as f:
            f.write(x)


def main():
    results = get_directory()
    for result in results:
        href = result['href']
        directory_image = result['image']
        directory_title = result['title']
        detail_list = get_zhibo(href, directory_title, directory_image)
        save_content(detail_list, directory_title)


if __name__ == '__main__':
    main()

 

你可能感兴趣的:(爬虫,python)