python爬取公众号历史文章_微信公众号爬虫--历史文章
今天搞了一个微信公众号历史文章爬虫的demo，亲测可行，记录一下！(不喜勿喷)
缺点：1.不是很智能 2. 兼容性不是很好，但是能应付正常情况啦
使用mysql+request
数据库部分
直接建表ddl吧：
CREATE TABLE `wechat_content` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`wechat_name` varchar(255) DEFAULT NULL COMMENT '公众号名字',
`title` varchar(225) DEFAULT NULL COMMENT '文章标题',
`content_url` varchar(1000) DEFAULT NULL COMMENT '文章地址',
`cover` varchar(1000) DEFAULT NULL COMMENT '封面图',
`source_url` varchar(1000) DEFAULT NULL COMMENT '转载url',
`source_name` varchar(255) DEFAULT NULL COMMENT '转载公众号名',
`datetime` varchar(255) DEFAULT NULL COMMENT '发布时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1629 DEFAULT CHARSET=utf8
def get_sqlConn():
try:
conn = pymysql.connect(
# host='localhost',
host = ip,
port=3306,
user='root',
password=密码,
db='py_mysql_test',
charset='utf8'
)
print('数据库连接成功！')
return conn
except:
print('error')
插入方法：
def insert_wechat_content(wechat_name,title,content_url,cover,source_url,source_name,datetime):
try:
conn = get_sqlConn()
cur = conn.cursor()
# sql = "INSERT INTO anjuke_beijing_onenum_all_house(house_name,house_plate,house_url,create_time) VALUES (%s,%s,%s,%s)"
sql = """INSERT INTO wechat_content(wechat_name,title,content_url,cover,source_url,source_name,datetime) VALUES (%s,%s,%s,%s,%s,%s,%s)""" % (wechat_name,title,content_url,cover,source_url,source_name,datetime)
print("微信公众号插入sql:%s" % sql)
cur.execute(sql)
conn.commit()
print('插入数据成功！')
except Exception as e:
print('插入发生数据错误！ERROR:%s' % e)
conn.rollback() # 回滚
finally:
cur.close()
conn.close()
print('操作数据库完毕！')
为了防止有重复的，还有一个查询方法：
def select_wechat_content(title):
conn = get_sqlConn()
cur = conn.cursor()
try:
sql = "SELECT EXISTS(SELECT 1 FROM wechat_content WHERE title=%s)" % title
print("微信公众号查询SQL：%s" % sql)
cur.execute(sql)
return cur.fetchall()[0]
except Exception as e:
print('查询发生数据错误！ERROR:%s' % e)
conn.rollback()
cur.close()
conn.close()
Python 爬取部分
首先分析一下，通过charles抓包可以看到历史文章的请求
/mp/profile_ext?action=getmsg&__biz=MzU0NDQ2OTkzNw==&f=json&offset=17&count=10&is_ok=1&scene=&uin=MTU4MzgxNjcwNg%3D%3D&key=5a37b8e9f2933463aa4c791beaedc828c781ae48f9a58c2067595d03e2a4da3d43e47af1b87aea58849a45838a5cd1375e69afd980a0562d3327ff9a7227684fa872ad73ae54f8d9ae5b2392595e0a4d&pass_ticket=n9Zz%2F2GEUA9SBL%2FLVdK8uLAPMwNph3rMVVksmgD0xrMOstqSxkc%2BaMVRVnfNAC9M&wxtoken=&appmsg_token=1030_sVyKhffomeHucF5TrTgG3CyPO9kX-j3obN4DNg~~&x5=0&f=json
这是历史文章的请求接口，通过分析我们得知需要动态获取的参数有：
__biz : 用户和公众号之间的唯一id，
uin ：用户的私密id
key ：请求的秘钥，一段时候只会就会失效。
offset ：偏移量
count ：每次请求的条数
我的做法是直接直接拿到请求的url，然后解析url中的参数，得到请求的params参数，方法见下：
__biz 应为后面又==，所以最后又重新赋值了
def get_parms(u):
data = u.split("&")
parms = {}
for i in data:
d = i.split("=")
parms[d[0]] = d[1]
parms['__biz'] = parms['__biz']+"=="
print(parms)
return parms
逻辑部分
def get_wx_article(u,wechat_name,index=0, count=10):
"""
:param u: 抓包获取的请求地址，不要/mp/profile_ext?
:param wechat_name: 公众号名，往数据库保存使用
:param index: 翻页
:param count: 每次请求条数
:return:
"""
offset = (index + 1) * count
url = "http://mp.weixin.qq.com/mp/profile_ext?"
params = get_parms(u)
params['offset'] = offset
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
'Cookie': 'rewardsn=; wxtokenkey=777; wxuin=1583816706; devicetype=Windows10; version=62070141; lang=zh_CN; pass_ticket=n9Zz/2GEUA9SBL/LVdK8uLAPMwNph3rMVVksmgD0xrMOstqSxkc+aMVRVnfNAC9M; wap_sid2=CILAnPMFElw5Z0w3VXRGdjhNTlF4Ujd0YXFUSjM0MUpkSGFkcUdHTC0wa08tcUR3aEtWZElvcGRwTnUtUjllbHRTU3ctZ0JJQkR0RW1TZjgwNVZZd1RCaTMwNkZSd1lFQUFBfjCjgIDtBTgNQJVO'
}
response = requests.get(url=url,params=params, headers=headers)
print(response.text)
resp_json = response.json()
if resp_json.get('errmsg') == 'ok':
resp_json = response.json()
# 是否还有分页数据， 用于判断return的值
can_msg_continue = resp_json['can_msg_continue']
# 当前分页文章数
msg_count = resp_json['msg_count']
general_msg_list = json.loads(resp_json['general_msg_list'])
list = general_msg_list.get('list')
print(list, "**************")
wechat_name = wechat_name
wechat_name = "'{}'".format(wechat_name)
for i in list:
print("=====>%s" % i)
if 'app_msg_ext_info' not in i:# 有特殊的公众号没有app_msg_ext_info字段，如果没有就跳过
continue
app_msg_ext_info = i['app_msg_ext_info']
# 标题
title = app_msg_ext_info['title']
title = "'{}'".format(title)
# 文章地址
content_url = app_msg_ext_info['content_url']
content_url = "'{}'".format(content_url)
# 封面图
cover = app_msg_ext_info['cover']
cover = "'{}'".format(cover)
# 转载路径
source_url = app_msg_ext_info['source_url']
source_url = "'{}'".format(source_url)
# 转载公众号
source_name = app_msg_ext_info['author']
source_name = "'{}'".format(source_name)
# 发布时间
datetime = i['comm_msg_info']['datetime']
datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(datetime))
datetime = "'{}'".format(datetime)
print(title,content_url)
print(source_url,source_name)
print(cover,datetime)
if select_wechat_content(title) == 1: # 防止数据重复
print("数据已经存在")
else:
insert_wechat_content(wechat_name,title,content_url,cover,source_url,source_name,datetime)
if can_msg_continue == 1:
return True
return False
else:
print('获取文章异常...')
return False
运行代码
if __name__ == '__main__':
index = 0
u = "action=getmsg&__biz=MzU0NDQ2OTkzNw==&f=json&offset=17&count=10&is_ok=1&scene=&uin=MTU4MzgxNjcwNg%3D%3D&key=5a37b8e9f2933463aa4c791beaedc828c781ae48f9a58c2067595d03e2a4da3d43e47af1b87aea58849a45838a5cd1375e69afd980a0562d3327ff9a7227684fa872ad73ae54f8d9ae5b2392595e0a4d&pass_ticket=n9Zz%2F2GEUA9SBL%2FLVdK8uLAPMwNph3rMVVksmgD0xrMOstqSxkc%2BaMVRVnfNAC9M&wxtoken=&appmsg_token=1030_sVyKhffomeHucF5TrTgG3CyPO9kX-j3obN4DNg~~&x5=0&f=json"
while 1:
print(f'开始抓取公众号第{index + 1} 页文章.')
flag = get_wx_article(u, "Python学习开发", index=index)
# 防止和谐，暂停8秒
time.sleep(8)
index += 1
if not flag:
print('公众号文章已全部抓取完毕，退出程序.')
break
print(f'..........准备抓取公众号第{index + 1} 页文章.')
最终效果：
后续可以再把首页的文章添加进去，敬请期待