抓取微博信息-Python程序-效果演示

代码效果演示
Gitee源码

# -*- coding: utf-8 -*-
# Version: Python 3.9.7
# Author: TRIX
# Date: 2021-10-04 17:36:05
# Use:抓取今日头条微博相关部分信息 并将信息储存到txt headers储存到json
from urllib.parse import urlencode
import requests
import pyperclip,json
from dateutil.parser import parse#处理时间


def getHeaders(noKeys=None,extraStripChars=[':'],doPrint=False,):
    '''noKeys 根据键名去除键值对 键名与键名间以一个space分割 忽略大小写 忽略是否strip

    extraStripChars 额外strip键名的无关字符 字符与字符间以一个space分割

    doPrint 是否打印headers和去除的键值对 只有在初次创建headers.json有用'''
    strs=pyperclip.paste()#读取剪贴字符串
    headers={}

    if 'Mozilla' not in strs:#如果复制字符串不是请求标头
        try:
            with open('headers.json','r',encoding='utf-8') as f:#读取储存的headers
                headers=json.loads(f.read())#json文本转dict对象
        except FileNotFoundError:
            return print('请至少完整复制一次请求标头信息\n复制方法：f12-网络-f5-名称-任意一个条目-标头-请求标头-复制所有')

    if not headers:
        def stripChars(word):#strip键名的无关字符
            word=word.strip()
            for char in extraStripChars:
                word=word.strip(char)
            return word

        keysValues=[]
        for keyOrValue in strs.split(': '):#以: 和\n切分字符串
            for kOrV in keyOrValue.split('\r\n'):
                keysValues.append(stripChars(kOrV))

        for i in range(0,len(keysValues),2):#生成headers字典
            headers[keysValues[i]]=keysValues[i+1]

        if noKeys:#去除指定键值对
            popKvs=[]
            for key in noKeys.split():
                popKvs.append(f"'{key}':'{headers.pop(stripChars(key))}'")

        json_data=json.dumps(headers,ensure_ascii=False,indent=2)#dict对象转json文本 ensure_ascii让输出结果是中文而不是unicode indent缩进字符个数 值只能是 dict list int float str bool None
        with open('headers.json','w',encoding='utf-8') as h:#储存json数据
            h.write(json_data)
        print('headers信息已储存至headers.json')

        if doPrint:#打印
            print('headers={')
            for k,v in headers.items():
                print(f"'{k}':'{v}',")
            print('}')
            if popKvs:
                print('\n去除headers的键值对如下：')
                for kv in popKvs:
                    print(kv)
                print('\n')
    return headers

def getPage(page):
    params={
    'uid':1618051664,
    'page':page,#通过page刷新微博
    'feature':0,
    }
    url='https://weibo.com/ajax/statuses/mymblog?'+urlencode(params)#构造url
    try:
        response=requests.get(url,headers=headers,)#请求数据
        print(f'正在尝试爬取数据：{url}响应码：{response.status_code}')
        if response.status_code==200:
            return response.json()
    except requests.ConnectionError as e:
        print('爬取失败：',e.args)

def getLongText(urlid):
    url='https://weibo.com/ajax/statuses/longtext?id='+urlid#构造url
    try:
        response=requests.get(url,headers=headers,)#请求数据
        print(f'检测到长文本 正在尝试爬取数据：{url}，响应码：{response.status_code}')
        if response.status_code==200:
            return response.json()
    except requests.ConnectionError as e:
        print('爬取失败：',e.args)

def getblogs(json):
    if json:#如果json!=None
        blogs=json.get('data').get('list')
        articleCount=len(blogs)
        articles=[]
        for blog in blogs:
            #print(blog)
            info={}#微博数据
            info['id']=blog.get('idstr')

            info['createTime']=str(parse(blog.get('created_at'))).split('+')[0]#处理时间格式

            if blog.get('isLongText'):#如果微博为长文本
                longText=getLongText(blog.get('mblogid'))
                if longText:
                    longText=longText.get('data')
                    info['text']=longText.get('longTextContent')

                    topicStruct=longText.get('topic_struct')
                    if topicStruct:
                        info['topics']=''
                        for topic in topicStruct:
                            info['topics']+=topic.get('topic_title')+' '
                    else:
                        info['topic']='无'
            else:
                info['text']=blog.get('text_raw')

            if blog.get('retweeted_status'):#如果微博含转载内容
                info['text']+='该微博含转载微博 内容略'

            if info.get('page_info'):#如果微博有视频或图片
                info['text']+='该微博含视频 或 图片 内容略'
            tagStruct=blog.get('tag_struct')
            if tagStruct:#如果有标签
                info['tags']=''
                for tag in tagStruct:
                    info['tags']+=tag.get('tag_name')+' '
            else:
                info['tags']='无'

            topicStruct=blog.get('topic_struct')
            if topicStruct:#如果该微博有主题
                info['topics']=''
                for topic in topicStruct:
                    info['topics']+=topic.get('topic_title')+' '
            else:
                info['topics']='无'

            info['likesCount']=blog.get('attitudes_count')
            info['commentsCount']=blog.get('comments_count')
            info['repostsCount']=blog.get('reposts_count')

            info['text']=info['text'].strip('')#替换一些不正常字符
            info['text']=info['text'].replace('>','>')

            if 'http' in info['text']:#如果有视频链接
                textAndUrl=info['text'].split('http')
                info['text']=textAndUrl[0]
                info['videoUrl']='http'+textAndUrl[-1]
            else:
                info['videoUrl']='无'

            articles.append(info)
        return articles

def getPageblogs(pages):#抓取pages页的所有微博的部分相关信息
    with open('blogs.txt','w',encoding='utf-8') as f:
        f.write('微博页面链接：{}\n'.format(headers['referer']))
        f.write(f'共抓取了{pages}页微博相关信息，抓取信息如下：\n\n')
        for n in range(pages):
            json=getPage(n+1)
            articles=getblogs(json)
            f.write(f'---第{n+1}页---\n')
            for i,a in enumerate(articles):
                f.write(f'---第{i+1}个微博---\n')
                f.write('微博id：{}\n'.format(a['id']))
                f.write('发布时间：{}\n'.format(a['createTime']))
                f.write('内容：{}\n'.format(a['text']))
                f.write('视频链接：{}\n'.format(a['videoUrl']))
                f.write('微博标签：{}\n'.format(a['tags']))
                f.write('微博主题：{}\n'.format(a['topics']))
                f.write('点赞数：{}\n'.format(a['likesCount']))
                f.write('评论数：{}\n'.format(a['commentsCount']))
                f.write('转载数：{}\n'.format(a['repostsCount']))
                f.write('\n')

headers=getHeaders(noKeys='path')
getPageblogs(3)