Python简单爬虫3

BeautifulSoup根据标签类型爬取微博热搜并写入文件

import requests
from bs4 import BeautifulSoup

headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}

url = 'https://s.weibo.com/top/summary?cate=realtimehot&sudaref=www.baidu.com&display=0&retcode=6102'

re = requests.get(url, headers=headers).text

#path = '新建文本文档.txt'
#with open(path, 'w', encoding='utf-8') as f:
 # f.write(re)

re = re.encode('utf-8')
#print (re)
soup = BeautifulSoup(re, 'lxml')

items= soup.find_all('td', attrs={'class': 'td-02'})

#print (items)
path = '新建文本文档.txt'
with open(path, 'w', encoding='utf-8') as f:
    cnt = 0
    for item in items:
        title = item.a.get_text()
        link = item.a['href']
        print ({'标题' : title, '链接' : link})
        string = str(cnt) + ' 标题: ' + title + '\n'+ '   链接: ' + link + '\n'
        cnt += 1
        f.write(string)