1 #!/usr/bin/env python
2
3 import cStringIO #
4 import formatter #
5 from htmllib import HTMLParser # We use various classes in these modules for parsing HTML.
6 import httplib # We only need an exception from this module
7 import os # This provides various file system functions
8 import sys # We are just using argv for command-line arguments
9 import urllib # We only need the urlretrieve()function for downloading Web pages
10 import urlparse # We use the urlparse()and urljoin()functions for URL manipulation
11
12 class Retriever(object):
13 __slots__ = ('url','file')
14
15 def __init__(self,url):
16 self.url, self.file = self.get_file(url)
17
18 def get_file(self, url, default='index.html'):
19 'Create usable local filename from URL'
20 parsed = urlparse.urlparse(url) # ParseResult(scheme='http', netloc='www.baidu.com', path='', params='', query='', fragment='')
21 host = parsed.netloc.split('@')[-1].split(':')[0] # 'www.baidu.com'
22 filepath = '%s%s' % (host,parsed.path) # 'www.baidu.com'
23 if not os.path.splitext(parsed.path)[1]: # ''
24 filepath = os.path.join(filepath, default) # 'www.baidu.com\\index.html'
25 linkdir = os.path.dirname(filepath) # 'www.baidu.com'
26 if not os.path.isdir(linkdir): # False
27 if os.path.exists(linkdir): # False
28 os.unlink(linkdir)
29 os.makedirs(linkdir) # make a directory named by link directory on the hard disc
30 return url, filepath
31
32 def download(self):
33 'Download URL to specific name file'
34 try:
35 retval = urllib.urlretrieve(self.url, self.file)
36 except (IOError, httplib.InvalidURL) as e:
37 retval = (('*** ERROR:bad URL "%s": %s' % (self.url,e)),)
38 return retval
39
40 def parse_links(self):
41 'Parse out the links found in downloaded HTML file'
42 f = open(self.file, 'r')
43 data = f.read()
44 f.close()
45 parser = HTMLParser(formatter.AbstractFormatter(formatter.DumbWriter(cStringIO.StringIO())))
46 parser.feed(data)
47 parser.close()
48 return parser.anchorlist
49
50 class Crawler(object):
51 count = 0 # the number of objects downloaded from the internet
52
53 def __init__(self, url):
54 self.q = [url] # a queue of links to download
55 self.seen = set() # a set containing all the links that we have seen(downloaded) already
56 parsed = urlparse.urlparse(url)
57 host = parsed.netloc.split('@')[-1].split(':')[0]
58 self.dom = '.'.join(host.split('.')[-2:]) # 'b.a.i.d.u'
59
60 def get_page(self, url, media=False):
61 'Download page & parse links, add to queue if nec'
62 r = Retriever(url)
63 fname = r.download()[0] # 'www.baidu.com\\index.html'
64 if fname[0] == '*': # 'w'
65 print fname, '... skipping parse'
66 return
67 Crawler.count += 1 # 1
68 print '\n(', Crawler.count, ')' # (1)
69 print 'URL:', url # URL: http://www.baidu.com
70 print 'FILE:', fname # FILE: www.baidu.com\\index.html
71 self.seen.add(url) # set(['http://www.baidu.com'])
72 ftype = os.path.splitext(fname)[1] # '.html'
73 if ftype not in ('.htm', '.html'): # False
74 return
75
76 for link in r.parse_links():
77 if link.startswith('mailto:'): # False
78 print '... discarded, mailto link'
79 continue
80 if not media: # False
81 ftype = os.path.splitext(link)[1]
82 if ftype in ('.mp3','.mp4','.m4v','.wav'):
83 print '... discarded, media file'
84 continue
85 if not link.startswith('http://'): # False
86 link = urlparse.urljoin(url, link)
87 print '*', link,
88 if link not in self.seen: # True
89 if self.dom not in link: # False
90 print '... discarded, not in domain'
91 else:
92 if link not in self.q:
93 self.q.append(link)
94 print '... new, added to Q'
95 else:
96 print '... discarded, already in Q'
97 else:
98 print '... discarded, already processed'
99
100 def go(self, media=False):
101 'Process next page in queue (if any)'
102 while self.q:
103 url = self.q.pop()
104 self.get_page(url, media)
105
106 def main():
107 if len(sys.argv) > 1:
108 url = sys.argv[1]
109 else:
110 try:
111 url = raw_input('Enter starting URL:')
112 except(KeyboardInterrupt, EOFError):
113 url = ''
114 if not url:
115 return
116 if not url.startswith('http://') and not url.startswith('ftp://'):
117 url = 'http://%s/' % url
118 robot = Crawler(url)
119 robot.go()
120
121 if __name__ == '__main__':
122 main()
