爬取knewone上的信息

爬取knewone上的信息

 author = 'LEE'
 
 
-- coding: utf-8 - 
 
 
from bs4 import BeautifulSoup
 import time
 import requests
 import time
 import io
 import sys
 import urllib.request
 sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')
 
 
url = 'https://knewone.com/things/?page='
 def get_page(url,data=None):
 wb_data = requests.get(url)
 soup = BeautifulSoup(wb_data.text,'lxml')
 imgs = soup.select('a.cover-inner > img')
 titles =soup.select('section.content > h4 > a')
 links = soup.select('section.content > h4 > a')
 #print(soup)
 if data ==None:
 for img,title,link in zip(imgs,titles,links):
 data = {
 'img':img.get('src'),
 'title':title.get('title'),
 'link':link.get('href'),
 }
 print(data)
 def get_more_page(start,end):
 for one in range(start,end):
 get_page(url+str(one))
 time.sleep(1)
 
 
get_more_page(1,10)