java如何实现大文件加速下载

针对大文件下载,Java可以使用多线程实现加速下载,并且结合断点续传功能,可以提高下载的稳定性和效率。具体实现步骤如下:

获取文件总长度和已下载长度,如果已下载长度等于文件总长度,则说明文件已下载完成,不需要进行下载。

如果已下载长度小于文件总长度,则需要进行多线程下载。根据文件总长度和线程数计算出每个线程需要下载的文件块大小,然后每个线程分别下载对应的文件块。

在HTTP请求中添加Range头信息,指定下载的起始位置和结束位置,例如Range:bytes=0-499表示下载文件的前500个字节。每个线程下载时需要指定不同的Range范围。

下载过程中,记录已下载的字节数和文件块的下载状态,并在下载中断时保存下载状态,以便下次继续下载。

下载完成后,将所有文件块合并成完整的文件。

下面是一个简单的Java实现多线程下载的示例代码:


import java.io.*;
import java.net.*;
import java.util.concurrent.*;

public class DownloadManager {
    private static final int THREAD_POOL_SIZE = 5;
    private final URL url;
    private final String savePath;
    private final int bufferSize;
    private final int threadCount;
    private final ExecutorService executorService;

    public DownloadManager(String url, String savePath, int bufferSize, int threadCount) throws MalformedURLException {
        this.url = new URL(url);
        this.savePath = savePath;
        this.bufferSize = bufferSize;
        this.threadCount = threadCount;
        this.executorService = Executors.newFixedThreadPool(threadCount);
    }

    public void download() throws IOException, InterruptedException {
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        int totalSize = conn.getContentLength();
        int blockSize = totalSize / threadCount;
        int downloadedSize = 0;
        boolean isCompleted = true;

        RandomAccessFile out = new RandomAccessFile(savePath, "rw");
        out.setLength(totalSize);
        out.close();

        for (int i = 0; i < threadCount; i++) {
            int start = i * blockSize;
            int end = (i == threadCount - 1) ? totalSize - 1 : (i + 1) * blockSize - 1;
            DownloadTask task = new DownloadTask(url, savePath, start, end, bufferSize);
            executorService.execute(task);
        }

        executorService.shutdown();
        executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);

        for (int i = 0; i < threadCount; i++) {
            int start = i * blockSize;
            int end = (i == threadCount - 1) ? totalSize - 1 : (i + 1) * blockSize - 1;
            File file = new File(savePath + "." + i);
            if (!file.exists() || file.length() != end - start + 1) {
                isCompleted = false;
                break;
            }
        }

        if (isCompleted) {
            FileOutputStream outStream = new FileOutputStream(savePath);
            for (int i = 0; i < threadCount; i++) {
                FileInputStream inStream = new FileInputStream(savePath + "." + i);
                byte[] buffer = new byte[bufferSize];
                int len;
                while ((len = inStream.read(buffer)) != -1) {
                    outStream.write(buffer, 0, len);
                }
                inStream.close();
                File file = new File(savePath + "." + i);
                file.delete();
            }
            outStream.close();
        }
    }

    private class DownloadTask implements Runnable {
        private final URL url;
        private final String savePath;
        private final int start;
        private final int end;
        private final int bufferSize;

        public DownloadTask(URL url, String savePath, int start, int end, int bufferSize) {
            this.url = url;
            this.savePath = savePath;
            this.start = start;
            this.end = end;
            this.bufferSize = bufferSize;
        }

        @Override
        public void run() {
            try {
                HttpURLConnection conn = (HttpURLConnection) url.openConnection();
                conn.setRequestMethod("GET");
                conn.setRequestProperty("Range", "bytes=" + start + "-" + end);

                InputStream in = conn.getInputStream();
                RandomAccessFile out = new RandomAccessFile(savePath + "." + Thread.currentThread().getId(), "rw");
                out.seek(start);

                byte[] buffer = new byte[bufferSize];
                int len;
                while ((len = in.read(buffer)) != -1) {
                    out.write(buffer, 0, len);
                }

                in.close();
                out.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

在上面的代码中,我们使用了ExecutorService线程池来管理多个下载线程。每个线程下载对应的文件块,并将下载状态保存在单独的文件中。在所有线程下载完成后,我们将所有文件块合并成完整的文件。

在使用时,可以创建DownloadManager对象,并调用download()方法启动下载。例如:


DownloadManager manager = new DownloadManager("http://example.com/largefile.zip", "C:/Downloads/largefile.zip", 1024, 5);
manager.download();

其中,第一个参数是要下载的文件URL,第二个参数是保存路径,第三个参数是缓冲区大小,第四个参数是线程数。



其他方法客供参考:


public void downLoadFile(HttpServletRequest request, HttpServletResponse response) throws IOException {
        // 设置编码格式
        response.setCharacterEncoding(UTF_8);
        //获取文件路径
        String fileName = request.getParameter("fileName");
        String path = OUTPUT_PATH;
        //参数校验
        log.info(fileName, path);
        //完整路径(路径拼接待优化-前端传输优化-后端从新格式化  )
        String pathAll = path + File.separator + fileName;
        log.info("pathAll{}", pathAll);
        Optional pathFlag = Optional.ofNullable(pathAll);
        File file = null;
        if (pathFlag.isPresent()) {
            //根据文件名,读取file流
            file = new File(pathAll);
            log.info("文件路径是{}", pathAll);
            if (!file.exists()) {
                log.warn("文件不存在");
                return;
            }
        } else {
            //请输入文件名
            log.warn("请输入文件名!");
            return;
        }
        try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
            //分片下载
            long fSize = file.length();//获取长度
            response.setContentType("application/x-download");
            String file_Name = URLEncoder.encode(file.getName(), "UTF-8");
            //获取下载文件名
            String newFileName = URLEncoder.encode(request.getParameter("sourceDept"), "UTF-8") + ".zip";
            response.addHeader("Content-Disposition", "attachment;filename=" + (StringUtils.isNotEmpty(newFileName) ? newFileName : fileName));
            //根据前端传来的Range  判断支不支持分片下载
            response.setHeader("Accept-Ranges", "bytes");
            response.setHeader("fName", file_Name);
            //定义断点
            long pos = 0, last = fSize - 1, sum = 0;
            //判断前端需不需要分片下载
            if (null != request.getHeader("Range")) {
                response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
                String numRange = request.getHeader("Range").replaceAll("bytes=", "");
                String[] strRange = numRange.split("-");
                if (strRange.length == 2) {
                    pos = Long.parseLong(strRange[0].trim());
                    last = Long.parseLong(strRange[1].trim());
                    //若结束字节超出文件大小 取文件大小
                    if (last > fSize - 1) {
                        last = fSize - 1;
                    }
                } else {
                    //若只给一个长度  开始位置一直到结束
                    pos = Long.parseLong(numRange.replaceAll("-", "").trim());
                }
            }
            long rangeLength = last - pos + 1;
            String contentRange = new StringBuffer("bytes").append(pos).append("-").append(last).append("/").append(fSize).toString();
            response.setHeader("Content-Range", contentRange);
            response.setHeader("Content-Length", String.valueOf(rangeLength));
            response.setHeader("Connection", "keep-alive");
            response.setHeader("Keep-Alive", "timeout=300, max=100");

            is.skip(pos);//跳过已读的文件(重点,跳过之前已经读过的文件)
            byte[] buffer = new byte[8192];  // 8KB buffer
            int length = 0;
            try (OutputStream os = new BufferedOutputStream(response.getOutputStream())) {
                while (sum < rangeLength) {
                    length = is.read(buffer, 0, (rangeLength - sum) <= buffer.length ? (int) (rangeLength - sum) : buffer.length);
                    sum = sum + length;
                    os.write(buffer, 0, length);
                }
            } // 这里的OutputStream将在这个块结束时关闭
            log.info("下载完成");
        } catch (IOException ex) {
            // handle exception here
            log.error("下载失败", ex);
        }
    }

你可能感兴趣的:(java如何实现大文件加速下载)