java多线程下载器

最近在看爬虫，其中的文件下载就找了下网上的资料，写了一个test，随便记下。

Header

import java.util.HashMap;
/**
 * @author xbl
 * @date 2018-09-18
 */
public class Header {
    public static final HashMap header = new HashMap<>();
    private Header() {
    }
    static {
        header.put("Accept-Encoding", "gzip, deflate");
        header.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        header.put("Cache-Control", "max-age=0");
        header.put("Connection", "keep-alive");
        header.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0");
        header.put("Accept",
                "image/gif,image/jpeg,image/pjpeg,image/pjpeg, "
                        + "application/x-shockwave-flash, application/xaml+xml, "
                        + "application/vnd.ms-xpsdocument, application/x-ms-xbap"
                        + "application/x-ms-application,application/vnd.ms-excel"
                        + "application/vnd.ms-powerpoint, application/msword,*/*");
    }

}

DownUtil

import entity.Header;

import java.io.RandomAccessFile;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;

/**
 * @author xbl
 * @date 2018-09-17
 */
public class DownUtil {
    //定义下载路径
    private String path;
    //指定所下载的文件的保存位置
    private String targetFile;
    //定义下载线程的数量
    private int threadNum;
    //定义下载线程的对象
    private DownloadThread[] threads;
    //下载文件的总大小
    private int fileSize;
    //线程计数器
    private CountDownLatch latch;

    public DownUtil(String path,String targetFile,int threadNum){
        this.path = path;
        this.targetFile = targetFile;
        this.threadNum = threadNum;
        threads = new DownloadThread[threadNum];
        latch = new CountDownLatch(threadNum);
    }
    public void downLoad() throws Exception{
        long t1 = System.currentTimeMillis();
        URL url = new URL(path);
        HttpURLConnection conn = (HttpURLConnection)url.openConnection();
        conn.setConnectTimeout(5 * 1000);
        //设置请求方法
        conn.setRequestMethod("GET");
        //设置请求属性
        Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
        //得到文件大小
        fileSize = conn.getContentLength();
        conn.disconnect();
        int currentPartSize = fileSize / threadNum + 1;
        RandomAccessFile file = new RandomAccessFile(targetFile,"rw");
        //设置本地文件大小
        file.setLength(fileSize);
        file.close();
        for(int i = 0;i < threadNum;i++){
            //计算每个线程的下载位置
            int startPos = i * currentPartSize;
            //每个线程使用一个RandomAccessFile进行下载
            RandomAccessFile currentPart = new RandomAccessFile(targetFile,"rw");
            //定位该线程的下载位置
            currentPart.seek(startPos);
            //创建下载线程
            threads[i] = new DownloadThread(startPos, currentPartSize, currentPart , path , latch);
            Thread t = new Thread(threads[i]);
            t.start();
        }
        BigDecimal n = BigDecimal.ZERO;
        while (true){
            BigDecimal length = BigDecimal.ZERO;
            if (latch.getCount()==0 && BigDecimal.ONE.compareTo(n)<=0){break;}
            for (DownloadThread downloadThread:threads) {
                length = length.add(new BigDecimal(downloadThread.getLength()));
            }
            n = length.divide(new BigDecimal(fileSize), 4, BigDecimal.ROUND_HALF_UP);
            System.out.println("已下载"+n.multiply(new BigDecimal("100"))+"%");
            Thread.sleep(1000);
        }
        long t2 = System.currentTimeMillis();
        System.out.println("下载完成，文件大小为"+fileSize*1.0/(1024*1024)+"M，共用时"+(t2-t1)*1.0/1000+"秒,平均"+fileSize*1.0/(1024*1024)/((t2-t1)/1000)+"MB/s");
        //线程等待,一个文件下载成功后继续下一个
//        latch.await();
    }
}

DownloadThread

import entity.Header;

import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;

/**
 * @author xbl
 * @date 2018-09-17
 */
public class DownloadThread implements Runnable {

    //当前线程的下载位置
    private int startPos;
    //定义当前线程负责下载的文件大小
    private int currentPartSize;
    //当前线程需要下载的文件块,此类的实例支持对随机访问文件的读取和写入。
    private RandomAccessFile currentPart;
    //定义该线程已下载的字节数
    private int length;
    //线程计数器
    private CountDownLatch latch;

    private String path;

    public DownloadThread(int startPos,int currentPartSize,RandomAccessFile currentPart, String path ,CountDownLatch latch){
        this.startPos = startPos;
        this.currentPartSize = currentPartSize;
        this.currentPart = currentPart;
        this.path = path;
        this.latch =latch;
    }
    @Override
    public void run(){
        InputStream inputStream = null;
        try{
            URL url = new URL(path);
            HttpURLConnection conn = (HttpURLConnection)url.openConnection();
            conn.setConnectTimeout(5 * 1000);
            //设置请求方法
            conn.setRequestMethod("GET");
            //设置请求属性
            Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
            inputStream = conn.getInputStream();
            //inputStream.skip(n);跳过和丢弃此输入流中数据的 n 个字节
            inputStream.skip(this.startPos);
            byte[] buffer = new byte[1024];
            int hasRead = 0;
            //读取网络数据写入本地
            while(length < currentPartSize && (hasRead = inputStream.read(buffer)) != -1){
                currentPart.write(buffer, 0, hasRead);
                length += hasRead;
            }
        }
        catch(Exception e){
            e.printStackTrace();
        }finally {
            try {
                currentPart.close();
                inputStream.close();
            }catch (Exception e){
                e.printStackTrace();
            }
            latch.countDown();
        }
    }

    public int getLength() {
        return length;
    }
}

测试

/**
 * @author xbl
 * @date 2018-09-14
 */
public class MyDowloadTest {
    public static void main(String[] args) {
        DownUtil downUtil = new DownUtil("http://122.246.9.250/67747A6C5D84B71E22A3C3CFE/03000A03005B9A34F27DF9800CB6C41096EB0F-B8A1-40F9-BE9D-77E1AD6E6E21.mp4?ccode=0502&duration=391&expire=18000&psid=12d0040bed59918644c5f644cc4ffd32&sp=&ups_client_netip=73cd74c8&ups_ts=1537256639&ups_userid=&utid=WqjDEjj3X1oCATy6JD6F%2Fbwk&vid=XMzgyMjM0MzUwNA%3D%3D&vkey=Be4882f87d72ed2b036aa6d16b12d2968&s=efbfbd135a731defbfbd&ali_redirect_domain=vali-dns.cp31.ott.cibntv.net","C:\\Users\\abc\\Desktop\\xbl\\img\\2.mp4",8);
        try {
            downUtil.downLoad();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

效果如下:

ok