Web_Crawler

import requests
from bs4 import BeautifulSoup

def trade_spider(max_page):
    page = 1
    i = 0
    while page <= max_page:
        url = 'https://www.thenewboston.com/search.php?type=0&sort=reputation&page=' + str(page)
        source_code = requests.get(url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text)
        for link in soup.findAll('a',{'class':'desc-title'}):
            href = 'https://www.thenewboston.com'+ link.get('href')
            title = link.string
            print(title)
            print(href)
            i += 1
            print('\n')
        page += 1
    print(i)


trade_spider(5)