python多线程下载oss图片

亲测通过

import urllib2,os
from HTMLParser import HTMLParser
import  urllib
import  oss2
import  MySQLdb
import socket
from multiprocessing import Pool
import  traceback
from multiprocessing.dummy import Pool as ThreadPool

def save_pic(url,filename):
    count=0
    while True:
        try:
            print filename,'begin...'
            urllib.urlretrieve(url, filename)
        except socket.timeout:
            print filename,'count:',count
            count+=1
        except Exception,e:
            print filename,'other',e
            count+=1
        else:
            print filename,'successfully!'
            break

if __name__ == "__main__":

    access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '**********')
    access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '**************')
    bucket_name = os.getenv('OSS_TEST_BUCKET', '**********')
    endpoint = os.getenv('OSS_TEST_ENDPOINT', '*************')

    host = 'localhost'
    user = 'root'
    pwd = 'root'
    db = 'test'
    conn = MySQLdb.connect(host,user,pwd,db,charset='utf8')
    cursor = conn.cursor()

    sql='select * from ks_table'

    img_dir = "D:\\images"

    if not os.path.isdir(img_dir):
        os.mkdir(img_dir)

    os.chdir(img_dir)
    print os.getcwd()
    url = ""

    try:
       # execute sql
       cursor.execute(sql)
       # fetch records

       results = cursor.fetchall()
       print 'Parent process %s.' % os.getpid()
       pool=ThreadPool(50)
       for row in results:
          xuexinid = row[0]
          questionurl=row[1]
          url = "http://"+questionurl.replace("@", ".*******.aliyuncs.com/");
          filename = os.path.basename(url)
          filename=xuexinid+'_'+filename

          pool.apply_async(save_pic,args=(url,filename))
       pool.close()
       pool.join()
       print 'all successfully!'
    except Exception,e:
       print "Error: unable to fecth data",e.message
       print  traceback.print_exc()


    cursor.close()
    conn.close()