Python 26 Programming Tutorial - How to Build a Web Crawler

import requests
from bs4 import BeautifulSoup

def get_info(max_page):
   page= 1
   while page<=max_page:
       url = r'http://www.buckyslockerroom.com/Wisconsin-Men-s-Apparel-s/180.htm?searching=Y&sort=5&cat=180&show=90&page=' + str(page)
       source_code = requests.get(url)
       plain_text= source_code.text
       txt = BeautifulSoup(plain_text)
       for link in txt.findAll('a', {'class':'v-product__title productnamecolor colors_productname'}):
           href = link.get('href')
           print(href)



get_info(1)

你可能感兴趣的:(Python 26 Programming Tutorial - How to Build a Web Crawler)