从某图片论坛中自动下载每页的每个子post的图片

I often goto some  picture bbs to see pictures ,I have needed to download the pictures from some good post. 

But I hate of saving it to local disk by WebBrowser. WE NEED TO DO IT AUTOMATICALLY.

Some picture bbs forum usually has a list of each post , and each post contain the pictures , with url in it's code file.

So we

1. download the list 
2. grab each post
3. analyst each post to grab the pictures' url
4. download the pictures by url

 

Here is the code in python do step 1-2 (htmladdr.py):

 #!/usr/bin/python

import re
import os
import sys
import funimg
# BBS url  (might need to change)
url_pre= ' http://xxxxx.com/bbs '
# sub html in hte list (might need to change)
p= ' <a\shref=\"thread-\d+-\d+-\d+.html\"> '

def usage():
     print  sys.argv[0]+ '  filepath '

def getpicfromhtml(htmlpath):

    text=open(htmlpath, ' r ').readlines()
     print  " try to analyst ", htmlpath
    line=0
     for element  in text:
        m=re.findall(p, element)
         if m:
             try:
                 for element_m  in m:
                     # get the real relative url in element_m (might need to change)
                    url=element_m[9:-2]
                    
                     print url
                    dirname=url
                     if  not os.path.exists(dirname):
                        os.makedirs(dirname)

                    os.chdir(dirname)
                    url_sub=url_pre+dirname
                     print  " get " ,url_sub
                    os.system( ' GET  '+ url_sub+ '  HTTP/1.1 '+  '  >  '+  ' htmlfile ')
                    funimg.downloadpic( ' ./htmlfile ')
                     print url_sub
                     # go back to parent (might need to change)
                    os.chdir( ' .. ')
             except Exception:
                 pass
     #     else:
     #         print 'find no match'

# get max 5 pages from bbs
htmlindex=1
while htmlindex<=5:
    htmlfilename= ' page '+str(htmlindex)
     # forum list url (might need to change)
    html_url=url_pre+ ' forum-4- '+str(htmlindex)+ ' .html '
     print html_url
     if  not os.path.exists(htmlfilename):
         # download the pagefile to a file name: pagex (x is stand for page index)
         print  ' GET  '+ html_url+ '  HTTP/1.1 '+  '  >  '+ htmlfilename
        os.system( ' GET  '+ html_url+ '  HTTP/1.1 '+  '  >  '+ htmlfilename)
    getpicfromhtml( ' ./page '+str(htmlindex))
    htmlindex=htmlindex+1

Here is the code in python do step 3-4 (funimg.py):

 #!/usr/bin/python

import re
import os
import string
import sys
# (might need to change)
p=re.compile( ' .*<img.*src=\".*.jpg\" ', re.I)

def downloadpic(filename):
     # read from the filename
    text=open(filename, ' r ').readlines()
     for element  in text:
        m=re.match(p,element)
         if m:
             # get the real url of pic from m 
            url=m.group()
            i=string.find(url, " src= ")            
            url_sub=url[i+4+1:-1]
            j=string.rfind(url, " / ")
            jpgfile=url[j+1:-1]
             if  not os.path.exists(jpgfile):
                 # max retry 3 times
                os.system( ' wget  '+url_sub+ '  --tries=3 & ' )
                 print url_sub
             else:
                 print  " exists already ", jpgfile

EOF