python3-网络数据下载

1、使用urllib

# import urllib
from urllib import request

print("downloading with urllib")
url = 'https://raw.githubusercontent.com/abidrahmank/OpenCV2-Python-Tutorials/master/data/left.jpg'
print ("downloading with urllib")

# 第一个参数网络地址；第二个参数 本地保存位置
request.urlretrieve(url,"./left2.jpg")

2、使用requests

import requests

print ("downloading with requests")
url = 'https://raw.githubusercontent.com/abidrahmank/OpenCV2-Python-Tutorials/master/data/left.jpg'
with open("./left2.jpg", "wb") as code:
   code.write(requests.get(url).content)

3、使用urllib3

如果没有urllib3，先执行 sudo pip3 install urllib3

import urllib3

print ("downloading with urllib3")
http=urllib3.PoolManager()
url = 'https://raw.githubusercontent.com/abidrahmank/OpenCV2-Python-Tutorials/master/data/left.jpg'
r=http.request('GET',url)

with open("./left2.jpg", "wb") as code:
   code.write(r.data)

4、下载文件并解压读取

# -*- coding: UTF-8 -*-

"""
下载一个压缩包文件，并执行解压
"""
import urllib3
import tarfile
import os
import shutil
import gzip


if not os.path.exists("./t10k-labels-idx1-ubyte.gz"):
   # 下载 文件
   print ("downloading with urllib3")
   http=urllib3.PoolManager()
   url = 'http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz'
   r=http.request('GET',url)

   with open("./t10k-labels-idx1-ubyte.gz", "wb") as code:
      code.write(r.data)

   with gzip.open('./t10k-labels-idx1-ubyte.gz', 'rb') as read, open('t10k-labels-idx1-ubyte', 'wb') as write:
      shutil.copyfileobj(read, write)

else:

   # 读取
   print('直接读取.gz文件')
   with gzip.open('./t10k-labels-idx1-ubyte.gz', 'rb') as read:
      data = read.read()
      print(data)

   # 解压 在读取
   print('先解压在读取')
   with gzip.open('./t10k-labels-idx1-ubyte.gz', 'rb') as read, open('t10k-labels-idx1-ubyte', 'wb') as write:
         shutil.copyfileobj(read, write)

   with open('t10k-labels-idx1-ubyte','rb') as read:
      data = read.read()
      print(data)

也可以使用shell脚本执行下载和解压，在命令终端输入： sh xxx.sh 执行命令

xxx.sh

#!/bin/bash

# 下载文件
wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz

# 解压
gunzip  t10k-labels-idx1-ubyte.gz

其他类型的压缩文件使用参考：http://blog.csdn.net/wc781708249/article/details/78192924

5、其他

# -*- coding:utf-8 -*-

'''
实现从网络上下载图片
'''

import urllib.request
import shutil

MODEL_URL='https://image.baidu.com/search/down?tn=download&word=download&ie=' \
          'utf8&fr=detail&url=https%3A%2F%2Ftimgsa.baidu.com%2Ftimg%3Fimage%26quality' \
          '%3D80%26size%3Db9999_10000%26sec%3D1520315813170%26di%3Df0cf0f7c8e041be005d78971c' \
          '5c239d5%26imgtype%3D0%26src%3Dhttp%253A%252F%252Fwww.bapimi.com%252Fuploads%252F0_' \
          '2501250005x2627376053_23.jpg&thumburl=https%3A%2F%2Fss2.bdstatic.com%2F70cFvnSh_' \
          'Q1YnxGkpoWK1HF6hhy%2Fit%2Fu%3D3436138697%2C1578547663%26fm%3D27%26gp%3D0.jpg'
model_path='./5555.jpg'
def download(model_path, verbose=1):
    """
    model_path: 存放的本地路径
    """
    if verbose > 0:
        print("Downloading pretrained model to " + model_path + " ...")
    with urllib.request.urlopen(MODEL_URL) as resp, open(model_path, 'wb') as out:
        shutil.copyfileobj(resp, out)
    if verbose > 0:
        print("... done downloading pretrained model!")

download(model_path)