python爬虫第一课,制作搜索引擎
from BeautifulSoup import *
from urlparse import urljoin
ignaorewords=set(['the','of','to','and','a','in','is','it'])我们的搜索引擎基于关键词, 所以将连词,冠词忽略
下面的代码是爬虫, 将网页的文本数据存储到我们的sqlite中, 大家看不懂也没有关系, 知道这些函数是干什么的就行了
from sqlite3 import dbapi2 as sqlite
import urllib2
class crawler:
def __init__(self,dbname):
self.con=sqlite.connect(dbname)
#连接并建立数据库, dbname 随意, 'xxx.db'就可以
def __del__(self):
self.con.close()
def dbcommit(self):
self.con.commit()
def getentryid(self,table,field,value,createnew=True):
cur=self.con.execute(
"select rowid from %s where %s='%s'" %(table,field,value))
res=cur.fetchone()
if res==None:
cur=self.con.execute(
"insert into %s (%s) values ('%s')" % (table,field,value))
return cur.lastrowid
else:
return res[0]
def addtoindex(self,url,soup):
if self.isindexed(url): return
print 'Indexing',url
#Get words
text=self.gettextonly(soup)
words=self.separatewords(text)
#Get URL id
urlid=self.getentryid('urllist','url',url)
# Link word to url
for i in range(len(words)):
word=words[i]
if word in ignaorewords: continue
wordid=self.getentryid('wordlist','word',word)
self.con.execute("insert into wordlocation(urlid,wordid,location) \
values(%d,%d,%d)" % (urlid,wordid,i))
def gettextonly(self,soup):
v=soup.string
if v==None:
c=soup.contents
resulttext=''
for t in c:
subtext=self.gettextonly(t)
resulttext+=subtext+'\n'
return resulttext
else:
return v.strip()
def separatewords(self,text):
splitter=re.compile('\\W*')
return [s.lower() for s in splitter.split(text) if s!='']
def isindexed(self,url):
u=self.con.execute(
"select rowid from urllist where url='%s'" % url).fetchone()
if u!=None:
#if crawled
v=self.con.execute(
'select * from wordlocation where urlid=%d' % u[0]).fetchone()
if v != None: return True
return False
def addlinkref(self,urlFrom,urlTo,linkText):
pass
def crawl(self,pages,depth=2):
for i in range(depth):
newpages=set()
for page in pages:
try:
c=urllib2.urlopen(page)
except:
print "Could not open",page
continue
soup=BeautifulSoup(c.read())
self.addtoindex(page,soup)
links=soup('a')
for link in links:
if 'href' in dict(link.attrs):
url=urljoin(page,link['href'])
if url.find("'") != -1:
continue
url=url.split('#')[0] #remove location portion
if url[0:4]=='http' and not self.isindexed(url):
newpages.add(url)
linkText=self.gettextonly(link)
self.addlinkref(page,url,linkText)
self.dbcommit()
pages=newpages
def createindextables(self):
self.con.execute('create table urllist(url)')
self.con.execute('create table wordlist(word)')
self.con.execute('create table wordlocation(urlid,wordid,location)')
self.con.execute('create table link(fromid integer,toid integer)')
self.con.execute('create table linkwords(wordid,linid)')
self.con.execute('create index wordidx on wordlist(word)')
self.con.execute('create index urlidx on urllist(url)')
self.con.execute('create index wordurlidx on wordlocation(wordid)')
self.con.execute('create index urltoidx on link(toid)')
self.con.execute('create index urlfromidx on link(fromid)')
self.dbcommit()pagelist=[['http://en.xjtu.edu.cn/'],
['http://www.lib.xjtu.edu.cn/'],
['http://en.wikipedia.org/wiki/Xi%27an_Jiaotong_University']]mycrawler=crawler('searchindex.db')
mycrawler.createindextables()mycrawler.crawl(pagelist[0])class searcher:
def __init__(self,dbname):
self.con=sqlite.connect(dbname)
def __del__(self):
self.con.close()
def getmatchrows(self,q):
# Strings to build the query
fieldlist='w0.urlid'
tablelist=''
clauselist=''
wordids=[]
# Split the words by spaces
words=q.split(' ')
tablenumber=0
for word in words:
#Get the word ID
wordrow=self.con.execute(
"select rowid from wordlist where word='%s'" % word).fetchone()
if wordrow!=None:
wordid=wordrow[0]
wordids.append(wordid)
if tablenumber>0:
tablelist+=','
clauselist+=' and '
clauselist+='w%d.urlid=w%d.urlid and ' % (tablenumber-1,tablenumber)
fieldlist+=',w%d.location' % tablenumber
tablelist+='wordlocation w%d' % tablenumber
clauselist+='w%d.wordid=%d' % (tablenumber,wordid)
tablenumber+=1
# Create the query from the separate parts
fullquery='select %s from %s where %s' % (fieldlist,tablelist,clauselist)
print fullquery
cur=self.con.execute(fullquery)
rows=[row for row in cur]
return rows,wordids
def geturlname(self,id):
return self.con.execute(
"select url from urllist where rowid=%d" % id).fetchone()[0]
def normaliszescores(self,scores,smallIsBetter=0):
vsmall=0.00001
if smallIsBetter:
minscore=min(scores.value())
return dict([(u,float(minscore)/max(vsmall,l)) for (u,l)\
in scores.items()])
else:
maxscore=max(scores.values())
if maxscore==0:
maxscore=vsmall
return dict([(u,float(c)/maxscore) for (u,c) in scores.items()])
#score methods
def frequencyscore(self,rows):
counts=dict([(row[0],0) for row in rows])
for row in rows:
counts[row[0]]+=1
return self.normaliszescores(counts)
def locationscore(self,rows):
locations=dict([(row[0],1000000) for row in rows])
for row in rows:
loc=sum(row[1:])
if loce=searcher('searchindex.db')e.query('xjtu college')1.000000 http://en.xjtu.edu.cn/XJTU_Introduction/Introduction.htm
0.941176 http://en.xjtu.edu.cn/info/1044/1683.htm
0.705882 http://en.xjtu.edu.cn/Schools_and_Colleges.htm
0.529412 http://en.xjtu.edu.cn/info/1044/1681.htm
0.470588 http://en.xjtu.edu.cn/Education/Undergraduate_Education.htm
0.382353 http://en.xjtu.edu.cn/XJTU_News/News.htm
0.382353 http://en.xjtu.edu.cn/Campus_Life/Student_Bodies.htm
0.294118 http://en.xjtu.edu.cn/XJTU_News/Teaching_and_learning.htm
0.294118 http://en.xjtu.edu.cn/info/1044/1572.htm
0.279412 http://en.xjtu.edu.cn/info/1044/1571.htm