Python多进程之进程池的使用

下载txt文件中的所有链接的图片

代码实现

from multiprocessing import Manager, Pool
import time
import os
import re

import requests

img_date_path = './img'

headers = {'Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'}


def open_url(q):
    # 取出链接
    if not q.full():  # 判断队列是否满
        with open('./url_202106031526.txt', 'r') as f:
            img_url = f.read().split()
        print(len(img_url))
        for i in range(len(img_url)):
            q.put(img_url[i])


def download(q):
    # 对链接进行下载
    while not q.empty():
        img_url = q.get(True, 2)
        response = requests.get(url=img_url, headers=headers, timeout=5, stream=True)
        a = re.findall('[u9fa5a-zA-Z0-9]+', img_url, re.S)  # 正则提出标点符号以外的所有内容
        name = "".join(a)
        print('正在下载%s' % name)
        with open("%s/%s.jpg" % (img_date_path, name), "wb") as f:
            f.write(response.content)
            time.sleep(0.01)


if __name__ == '__main__':
    t1 = time.time()
    print("(%s) start" % os.getpid())
    q = Manager().Queue()
    po = Pool(8)
    po.apply_async(open_url, (q,))
    # 先让上面的任务向Queue存入数据，然后再让下面的任务开始从中取数据
    for i in range(5):
        po.apply_async(download, (q,))
    po.close()
    po.join()
    print("(%s) End" % os.getpid())
    t2 = time.time()
    print(t2 - t1)

文件链接

https://i0.hdslb.com/bfs/album/26e0484fc12bf6c4294704c319ef59b86cfedafe.jpg
https://i0.hdslb.com/bfs/album/6c4bf2efc66c1f18e59c18b8d7dbf4cd053afdae.jpg
https://i0.hdslb.com/bfs/album/335dc3bd740c9182aa307f50f5f23d7859067417.jpg
https://i0.hdslb.com/bfs/album/87808a027d81fde62ce0360148c7c45368fa1b8c.jpg
https://i0.hdslb.com/bfs/album/9c6588c64e0281ec14c70b00c426c6451bf2bfc2.jpg
https://i0.hdslb.com/bfs/album/d98596c526be43446a45526d3371507818927f0d.jpg
https://i0.hdslb.com/bfs/album/23aeeb332a3b07ab2d4e436695a8c1741a13733a.jpg
https://i0.hdslb.com/bfs/album/c52022d4a2f4c0c5e7ad58310803da9cf097fbc2.jpg
https://i0.hdslb.com/bfs/album/9549410bc6fa04653fc2f20f2849e2b141bc1775.jpg
https://i0.hdslb.com/bfs/album/377e95a1f3e8b0640c28ecd5c46a1cc47d77da3a.jpg
https://i0.hdslb.com/bfs/album/a75db5253d98fbd6880dca5475c46befda5d400c.jpg
https://i0.hdslb.com/bfs/album/58311c63bc27925ba6a9672a70ac0623cdb04bfe.jpg
https://i0.hdslb.com/bfs/album/782c0606820f064060ce3099abcae97622c3dc62.png
https://i0.hdslb.com/bfs/album/9a137c2ec688cf4a78964aafd8e2942bf4606333.jpg
https://i0.hdslb.com/bfs/album/b568f211ca246549cc91edb7e1e1ad75321d0dbe.jpg
https://i0.hdslb.com/bfs/album/3042ae79f0958d497f1326ef95dd77cab8dba57e.jpg
https://i0.hdslb.com/bfs/album/9a849f7cc548d70d92a3d4eb5959cee12bc868d4.jpg
https://i0.hdslb.com/bfs/album/e09b32860f79840c6b42529fa67b47676a4c3d15.jpg
https://i0.hdslb.com/bfs/album/39f63bd7540ffe503df252c09108abb06aa9f9c6.jpg
https://i0.hdslb.com/bfs/album/18b9d7aece1773d36d1b57384fd8c9312d3b7bf7.jpg
https://i0.hdslb.com/bfs/album/c3b9d28bc118ab176675385944de0995f4b93896.jpg
https://i0.hdslb.com/bfs/album/705236978f71b44a50cc31bf55e34d7d09628de5.jpg
https://i0.hdslb.com/bfs/album/f5ca484eb2dc059311804857aa88d83f718f3d42.jpg
https://i0.hdslb.com/bfs/album/06bf054ffba43a8ffd971cb5764e0b54fc6a4e29.jpg
https://i0.hdslb.com/bfs/album/ec7c6938f2a57a2299263472e6b8ec7b0238cbbb.jpg
https://i0.hdslb.com/bfs/album/4cd9fe7d010ac5e2ec3d6acf2a012144fe988fcd.jpg
https://i0.hdslb.com/bfs/album/eb1337e10745f1c0c4803c91d04ceacb53c3d609.jpg
https://i0.hdslb.com/bfs/album/977df721dd4c3b2fbb0bd11e6a5c2f6f0ee4346c.jpg
https://i0.hdslb.com/bfs/album/3e88b29384689480f0279bd7352b45d885b82242.jpg
https://i0.hdslb.com/bfs/album/8530da35958e8ae500b08693767d3ab2f36244a1.jpg
https://i0.hdslb.com/bfs/album/e70d2a30411aaf07f439b1e9f496724e6b46ce11.jpg
https://i0.hdslb.com/bfs/album/f1e1ab21c49fe42d58d20b39ae6ee0eb130c0c88.jpg
https://i0.hdslb.com/bfs/album/8a4ae33d35cd8d2d45cd2524ce43f6459748b9dc.jpg
https://i0.hdslb.com/bfs/album/5f2f8ad43e8696c3daf96526be33267dd97de492.jpg
https://i0.hdslb.com/bfs/album/e57724be8c023b8ddcb335bd31629e62f3f350d7.jpg
https://i0.hdslb.com/bfs/album/bfb0e7c68fe8559ccdda0221ee4e9ad21b0138f9.jpg
https://i0.hdslb.com/bfs/album/0bdd96eaeea5cf1f24ba77f2640c28c48a29be6a.jpg
https://i0.hdslb.com/bfs/album/e432f8e76fb5a6e94dddddbdcb3206360a89becb.jpg
https://i0.hdslb.com/bfs/album/a0fbc1aac64d4b4e620da68ba024f64b06a7f640.jpg
https://i0.hdslb.com/bfs/album/cdb033287e6f603ae8fd7643a8ef05fad952c022.jpg
https://i0.hdslb.com/bfs/album/3953557d353a5af0ebdc0d60064fd9d9b5e3d4b1.jpg
https://i0.hdslb.com/bfs/album/d9cc5b439709e6a908b2368d0cd7f1f04e162ba8.jpg
https://i0.hdslb.com/bfs/album/d4ae714f7911b76ad87a34be42f66cb6d847d23c.jpg
https://i0.hdslb.com/bfs/album/e2836d067050c1a00d53537fa9eabec4871ddfb9.jpg
https://i0.hdslb.com/bfs/album/442379a58cf820fe1794d4bdef762ccbffd760b7.jpg
https://i0.hdslb.com/bfs/album/b568f211ca246549cc91edb7e1e1ad75321d0dbe.jpg
https://i0.hdslb.com/bfs/album/ebf7fb86515c5dfae5dc43ba2201693237f765ec.jpg
https://i0.hdslb.com/bfs/album/b5301728f804965e93d58dbd78a86dadd5225ebb.jpg
https://i0.hdslb.com/bfs/album/52ca01755f5778cc0ff7e4f8a28ee916ef6f480f.jpg
https://i0.hdslb.com/bfs/album/6775520a2b26ed51a3efc388236ce4bb41e96d46.jpg
https://i0.hdslb.com/bfs/album/3ec07e0a97687bb448d4d03af896c607dab2b841.jpg
https://i0.hdslb.com/bfs/album/dbac5552f27be7e2a284aaeeb3f7547029e775ec.jpg
https://i0.hdslb.com/bfs/album/8386c47e8e80ce17f8da126ea6276fc810485f8b.jpg
https://i0.hdslb.com/bfs/album/b8f65026e52a6bd6e40015b71eac9d707ee45eb7.jpg
https://i0.hdslb.com/bfs/album/e55a58f1bc54973dc9f9eb72161cbab3f19ef998.jpg
https://i0.hdslb.com/bfs/album/2603fd6fd3af415121b53c0cbc706886fc55f47a.jpg
https://i0.hdslb.com/bfs/album/a8ef2ecfa7045d61981ddd3870b138724db2f47c.jpg
https://i0.hdslb.com/bfs/album/400006ef0b173e9316a97523c6a30e9802ea2ed4.jpg
https://i0.hdslb.com/bfs/album/20849ce29f11ea7683e14d3a0b690a6ec110da04.jpg
https://i0.hdslb.com/bfs/album/3361a3b12c03f6e2a7d904db440043416d8103c0.jpg
https://i0.hdslb.com/bfs/album/ce0aafb110a11386690b86d9d7a8a6d9031768db.jpg
https://i0.hdslb.com/bfs/album/4cd062a44ba8d7a97949a08f9827daa8cb4243d1.jpg
https://i0.hdslb.com/bfs/album/64a45ad7cb05aa02267723fb958fc9a5e20ac2d1.jpg
https://i0.hdslb.com/bfs/album/ee811573286707043afffad83cff38560b5e1f51.jpg
https://i0.hdslb.com/bfs/album/b919a355369cdd29dd9ebe34ecb2f10914ea8c7c.jpg
https://i0.hdslb.com/bfs/album/c6623bf8056e337fe48c5f8e66208319fe5932f3.jpg
https://i0.hdslb.com/bfs/album/26c74e244d75daec13544b7aee199b8165704fc9.jpg
https://i0.hdslb.com/bfs/album/b2e7dc1367377e0de2f93566d0874b7b4a02d827.jpg
https://i0.hdslb.com/bfs/album/328f6301c9d31ff2f7a6e048027f734e4258b5a8.jpg
https://i0.hdslb.com/bfs/album/2a9469d862bc342c1787c520cb4bbca280806bc3.jpg
https://i0.hdslb.com/bfs/album/e08ccee56b31b7b2d9c9fca1306988674126bf70.jpg
https://i0.hdslb.com/bfs/album/2f1ea03a0211fd707fdb6e5897a4a3cef10530a1.jpg
https://i0.hdslb.com/bfs/album/5d7863d20c561645f0b8c74677d0c69950c4bb27.jpg
https://i0.hdslb.com/bfs/album/622855196a08bb83437f343a9be4e81574527bae.jpg
https://i0.hdslb.com/bfs/album/f8115a98504fc4abfa4abdd78ed0ff59cc6d4e44.jpg
https://i0.hdslb.com/bfs/album/83a25fa994cb9518cbe13763233084c7d03e6a5d.jpg
https://i0.hdslb.com/bfs/album/3175167bbc09504b14eb048987c20ead63be6a75.jpg
https://i0.hdslb.com/bfs/album/6f1ed32b6fb5774e327ab40d0b087bb85c6b3b89.jpg
https://i0.hdslb.com/bfs/album/6976975a11c87c020bad060eafe816ae9637e72f.jpg
https://i0.hdslb.com/bfs/album/6fbf790bb36b2ff3f2e4be073a6dc2847e146234.jpg
https://i0.hdslb.com/bfs/album/d711b9380195313b9fcea8cf11cb4f9d49c7ed78.jpg
https://i0.hdslb.com/bfs/album/a366414073d0a7625073b02a403086f89a419064.jpg
https://i0.hdslb.com/bfs/album/5d665b3dc45758c1de8cfa5f4370fbb1b119bf69.jpg
https://i0.hdslb.com/bfs/album/08c6d41b5cbd235e85426ea6deaccc7233632982.jpg
https://i0.hdslb.com/bfs/album/5cf217297eb9b92b38b77c57571cce714354ec97.jpg
https://i0.hdslb.com/bfs/album/bf55a30a5188b0dcc927de4e5aa97c1dc1f396d9.jpg
https://i0.hdslb.com/bfs/album/e2abef8aba641090e380b5405386bb1952046888.jpg
https://i0.hdslb.com/bfs/album/81394187a0abafdc9c87c9f44161c0c9f3733cdf.jpg
https://i0.hdslb.com/bfs/album/41a7edf2ed243230942cefa5f15eca8c56cd25ae.jpg
https://i0.hdslb.com/bfs/album/87915c33322e40babc920fce8583d273d368a168.png
https://i0.hdslb.com/bfs/album/0c64561346bd4acae851eb5fec7980c62b106be2.jpg
https://i0.hdslb.com/bfs/album/b6c3a9db2439ea21b85286cb0faf3d890b6e3a6a.jpg
https://i0.hdslb.com/bfs/album/3bb43fed5eda77f2739e85e220165b53fffc6d87.jpg
https://i0.hdslb.com/bfs/album/2ca29c58f5d0b31458bcc7f03dce2509e4f108da.jpg
https://i0.hdslb.com/bfs/album/4c09401267c50b177432893101332171bc05b1dd.jpg
https://i0.hdslb.com/bfs/album/bd49c9fed23a7ce816f04f9f237e549519f4b55c.jpg
https://i0.hdslb.com/bfs/album/48d55febd00ecfde3a79a3c7fe0f7178fc87e184.gif
https://i0.hdslb.com/bfs/album/7f7602a06edc62961918426ca7a15f9d72dc0748.jpg
https://i0.hdslb.com/bfs/album/a39639588b16a6cf4518f9c6dcf4adab84f94883.jpg
https://i0.hdslb.com/bfs/album/998464163b70ad904777ddaf35961acb6b0c77bd.jpg
https://i0.hdslb.com/bfs/album/505238887525c4786b16b63e563349167f4f4f69.jpg
https://i0.hdslb.com/bfs/album/7a053d2f38f64c5fa51ebf9ac90c018bdf2840e4.jpg
https://i0.hdslb.com/bfs/album/9f41f4347f06968d9f0bd275aaa17c6ae9a578ca.jpg
https://i0.hdslb.com/bfs/album/07d2781be53cd13253a988a40636d710f6e22289.jpg
https://i0.hdslb.com/bfs/album/08dcb7c3ba33c65aabd6c9a2964414e2580ec31a.jpg
https://i0.hdslb.com/bfs/album/9517cd5addfe904c2fea4ab0bedca10c3b733be0.jpg
https://i0.hdslb.com/bfs/album/9fec8760eda9a9964c0289e8ed90110c0448fe86.jpg
https://i0.hdslb.com/bfs/album/f081a9f3b32337318c8b4ed5477766ff3fce8469.jpg
https://i0.hdslb.com/bfs/album/523517c389f0ce65568bf97ff266748877827029.jpg
https://i0.hdslb.com/bfs/album/24e689dd23595585bc72e7f44805a1ab517ef41e.jpg
https://i0.hdslb.com/bfs/album/de5dd1eb2cc12bd6f87e2f94a2638ea3ab34af4f.jpg
https://i0.hdslb.com/bfs/album/bc70168318804cf5a59389e340b93e2c3c2a5bf5.jpg
https://i0.hdslb.com/bfs/album/dabb601daf257c05104d794b7562ba87c9a7f73c.jpg
https://i0.hdslb.com/bfs/album/1ebec1a365f8f579eefeff74bbcab739ae7dc77d.jpg
https://i0.hdslb.com/bfs/album/1c7a317c1a84bfec14c949f000ae7aa1de8bba3c.jpg
https://i0.hdslb.com/bfs/album/ee638db5b520882ee8789929f2f4229e8a7eaaae.jpg
https://i0.hdslb.com/bfs/album/0c24a795568013e44a3acaec706400b8cdee42cd.jpg
https://i0.hdslb.com/bfs/album/28332b089a64139a11ac6a0efc7d67f41178279c.jpg
https://i0.hdslb.com/bfs/album/bb5dbe14768f2488cb769ecf52b7a3b3f7992ef7.jpg
https://i0.hdslb.com/bfs/album/9d9a4ca2c62733b53b8d17a943f7b1cbe1a329b8.jpg
https://i0.hdslb.com/bfs/album/8d818cd8741e1f417dff541ed1d79128a76e7bfe.jpg
https://i0.hdslb.com/bfs/album/ea2df3b138ba6c1a994da091f526f09dfc1bd965.jpg
https://i0.hdslb.com/bfs/album/8c961abe6a6c14d523028d0fa73b855cff1ba71f.jpg
https://i0.hdslb.com/bfs/album/f2c49e132e645ae1dc1751a76904e240470680ff.jpg
https://i0.hdslb.com/bfs/album/60267d4357a368dd9de38d9cec39c6ddc3606f3e.jpg
https://i0.hdslb.com/bfs/album/33d70e63ac5c05bfaf30fa8dd5f1496528447652.jpg