python爬虫（2.获取网页外链与内链）

from urllib.request import urlopen
from urllib.parse import urlparse
from bs4 import BeautifulSoup
import re
import datetime
import random

pages = set()
random.seed(datetime.datetime.now())

#获取页面内链
def getInternalLinks(bsObj,includeUrl):
    includeUrl=urlparse(includeUrl).scheme+"://"+urlparse(includeUrl).netloc
    internalLinks=[]
    for link in bsObj.findAll("a",href=re.compile("^(/|.*"+includeUrl+")")):
        if link.attrs["href"] is not None:
            if link.attrs["href"] not in internalLinks:
                internalLinks.append(link.attrs['href'])
    return internalLinks

#获取页面外链并且不包含当前url的链接  "^(http|www)((?!"+excludeUrl+").)*$"
def getExternalUrl(bsObj,excludeUrl):
    externalUrl=[]
    for link in bsObj.findAll("a",re.compile("^(www|http)((?!"+excludeUrl+").)*$")):
        if link.attrs["href"] is not None:
            if link.attrs["href"] not in externalUrl:
                externalUrl.append(link.attrs['href'])
    return externalUrl

#主页面的外链，若无就找主页面中内链随机一个，再在该内链中找外链
def getRandomExternalLink(startingPage):
    html=urlopen(startingPage)
    bsObj=BeautifulSoup(html)
    externalLinks=getExternalUrl(bsObj,startingPage)
    if len(externalLinks)==0:
        domain = urlparse(startingPage).scheme+"://"+urlparse(startingPage).netloc
        print("No external links, looking around the site for one")
        InternalLinks=getInternalLinks(bsObj,domain)
        page="http://"+InternalLinks[random.randint(0,len(InternalLinks)-1)]
        getRandomExternalLink(page)
    else:
        return  externalLinks[random.randint(0, len(externalLinks)-1)]


def followExternalOnly(startingSite):
    externalLinks=getRandomExternalLink(startingSite)
    print("tha random external page is:"+externalLinks)
    followExternalOnly(externalLinks)