java 多线程下载文件

      关于下载文件,首先需要了解一点io和http请求,线程等相关知识,然后才能一步一步的趴坑整理出一个较为满意的结果。原先我写过一个较为简单的下载是单线程的,如下

package cn.zectec.hamster.baseservice.utils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.*;
import java.util.HashMap;
import java.util.Map;

public class DownloadUtil {
   private static Logger logger = LoggerFactory.getLogger(DownloadUtil.class);

   public static void errorFilePath(HttpServletResponse response, String msg){
      //没有该消息记录直接返回无此文件
      try {
         ServletOutputStream out=response.getOutputStream();
         OutputStreamWriter ow=new OutputStreamWriter(out,"UTF-8");
         ow.write(msg);
         ow.flush();
         ow.close();
      } catch (IOException e) {

      }catch (Exception e) {
         e.printStackTrace();
      }

   }
   /**
    * @param filePath 要下载的文件路径
    * @param returnName 返回的文件名
    * @param response HttpServletResponse
    * @param delFlag 是否删除文件
    */
   protected void download(String filePath,String returnName,HttpServletResponse response,boolean delFlag){
      prototypeDownload(new File(filePath), returnName, response, delFlag);
   }


   /**
    * @param file 要下载的文件
    * @param returnName 返回的文件名
    * @param response HttpServletResponse
    * @param delFlag 是否删除文件
    */
   public static void download(File file, String returnName, HttpServletResponse response, boolean delFlag){
      prototypeDownload(file, returnName, response, delFlag);
   }
   
   /**
    * @param file 要下载的文件
    * @param returnName 返回的文件名
    * @param response HttpServletResponse
    * @param delFlag 是否删除文件
    */
   private static void prototypeDownload(File file, String returnName, HttpServletResponse response, boolean delFlag){
      // 下载文件
      FileInputStream inputStream = null;
      ServletOutputStream outputStream = null;
      try {
         if(!file.exists()) return;
         response.reset();
         //设置响应类型   PDF文件为"application/pdf",WORD文件为:"application/msword", EXCEL文件为:"application/vnd.ms-excel"。
         response.setContentType("application/octet-stream;charset=utf-8");
         //设置响应的文件名称,并转换成中文编码
         returnName = URLEncoder.encode(returnName,"UTF-8");
         returnName = response.encodeURL(new String(returnName.getBytes(),"iso8859-1"));    //保存的文件名,必须和页面编码一致,否则乱码

         //attachment作为附件下载;inline客户端机器有安装匹配程序,则直接打开;注意改变配置,清除缓存,否则可能不能看到效果
         response.addHeader("Content-Disposition",   "attachment;filename="+returnName);

         //将文件读入响应流
         inputStream = new FileInputStream(file);
         outputStream = response.getOutputStream();
         int length = 1024;
         int readLength=0;
         byte buf[] = new byte[1024];
         readLength = inputStream.read(buf, 0, length);
         while (readLength != -1) {
            outputStream.write(buf, 0, readLength);
            readLength = inputStream.read(buf, 0, length);
         }
      }catch (IOException e) {
         errorFilePath(response,"无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }finally {
         try {
            outputStream.flush();
         } catch (IOException e) {

         }
         try {
            outputStream.close();
         } catch (IOException e) {

         }
         try {
            inputStream.close();
         } catch (IOException e) {

         }
         //删除原文件

         if(delFlag) {
            file.delete();
         }
      }
   }

   /**
    * by tony 2013-10-17
    * @param byteArrayOutputStream 将文件内容写入ByteArrayOutputStream
    * @param response HttpServletResponse 写入response
    * @param returnName 返回的文件名
    */
   public void download(ByteArrayOutputStream byteArrayOutputStream, HttpServletResponse response, String returnName) throws IOException{
      response.setContentType("application/octet-stream;charset=utf-8");
      returnName = response.encodeURL(new String(returnName.getBytes(),"iso8859-1"));          //保存的文件名,必须和页面编码一致,否则乱码
      response.addHeader("Content-Disposition",   "attachment;filename=" + returnName);  
      response.setContentLength(byteArrayOutputStream.size());
      
      ServletOutputStream outputstream = response.getOutputStream(); //取得输出流
      byteArrayOutputStream.writeTo(outputstream);               //写到输出流
      byteArrayOutputStream.close();                         //关闭
      outputstream.flush();                                //刷数据
   }

   /***
    * 浏览器远程下载文件
    */
   public static void downloadFileByHttpUrl(HttpServletResponse response,String urlStr,String fileName){
      OutputStream out = null;
      InputStream ips = null;
      try {
         logger.info("第一次请求url==>" + urlStr);
         URL url = new URL(urlStr);
         HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
         httpUrlConn.addRequestProperty("Content-type", "application/json");
         //设置请求超时时间
         httpUrlConn.setReadTimeout(15000);
         httpUrlConn.setConnectTimeout(15000);
         httpUrlConn.setDoOutput(true); //允许写出
         httpUrlConn.setDoInput(true);//允许写入
         httpUrlConn.setUseCaches(false);//不使用缓存
         // 设置请求方式(GET/POST)
         httpUrlConn.setRequestMethod("GET");
         httpUrlConn.connect();
         int responseCode = httpUrlConn.getResponseCode();
         if(responseCode!=200){
            errorFilePath(response,"无此文件");
         }else {
            ips = httpUrlConn.getInputStream();//字节流 输入流
            response.reset();
            response.setContentType("application/x-download");
            response.addHeader("Content-Disposition","attachment;filename="+ new String(fileName.getBytes(),"iso-8859-1"));
            response.setContentType("application/octet-stream");
            out = new BufferedOutputStream(response.getOutputStream());
            int length = 1024;
            int readLength=0;
            byte buf[] = new byte[1024];
            readLength = ips.read(buf, 0, length);
            while (readLength != -1) {
               out.write(buf, 0, readLength);
               readLength = ips.read(buf, 0, length);
            }
         }
      }catch (IOException e) {
         errorFilePath(response,"无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }finally {
         try {
            out.flush();
         } catch (IOException e) {

         }
         try {
            out.close();
         } catch (IOException e) {

         }
         try {
            ips.close();
         } catch (IOException e) {

         }
      }
   }

   /***
    * http 请求获取下载文件保存到本地
    * urlStr 远程地址
    * filePath 指定本地保存路径
    */
   public static Map downloadFileByHttpUrl(String urlStr, String filePath){
      Map map = new HashMap<>();
      map.put("success",false);
      OutputStream out = null;
      InputStream ips = null;
      String requestMethod="GET";
      try {
         File file = new File(filePath);
         logger.info("===================:"+urlStr);
         logger.info("===================获取流媒体服务中的文件:"+filePath);
         URL url = new URL(urlStr);
         HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
         httpUrlConn.addRequestProperty("Content-type", "application/json");
         //httpUrlConn.addRequestProperty("accessToken",accessToken);
         //设置请求超时时间
         httpUrlConn.setReadTimeout(30000);
         httpUrlConn.setConnectTimeout(30000);
         httpUrlConn.setDoOutput(true); //允许写出
         httpUrlConn.setDoInput(true);//允许写入
         httpUrlConn.setUseCaches(false);//不使用缓存

         // 设置请求方式(GET/POST)
         httpUrlConn.setRequestMethod(requestMethod);
         if ("GET".equalsIgnoreCase(requestMethod)) {
            httpUrlConn.connect();
         }
         // 将返回的输入流转换成字符串
         ips = httpUrlConn.getInputStream();
         out = new FileOutputStream(file,false);
         int length = 1024;
         int readLength=0;
         byte buf[] = new byte[1024];
         readLength = ips.read(buf, 0, length);
         while (readLength != -1) {
            String s = new String(buf, "UTF-8");
            if (s.contains("未找到")) {
               logger.info("==================putIOToResponseByHttp错误信息:"+s);
               file.delete();
               map.put("msg",s);
               return map;
            }else{
               out.write(buf, 0, readLength);
               readLength = ips.read(buf, 0, length);
            }
         }
         map.put("success",true);
      }catch (IOException e) {
         logger.info("无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }finally {
         try {
            out.flush();
         } catch (IOException e) {

         }
         try {
            out.close();
         } catch (IOException e) {

         }
         try {
            ips.close();
         } catch (IOException e) {

         }
      }
      return map;
   }

   /***
    * 根据远程http地址将流存到respons 返回失败或者成功
    * @param response 浏览器response
    * @param urlStr 远程http地址
    * @return 返回失败或者成功
    */
   public static boolean putIOToResponseByHttp(HttpServletResponse response, String urlStr) {
      OutputStream out = null;
      InputStream ips = null;
      String requestMethod="GET";
      try {
         if (StringUtils.isEmpty(urlStr)) {
            return false;
         }else{
               URL url = new URL(urlStr);
               HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
               httpUrlConn.addRequestProperty("Content-type", "application/json");
               //httpUrlConn.addRequestProperty("accessToken",accessToken);
               //设置请求超时时间
               httpUrlConn.setReadTimeout(300000);
               httpUrlConn.setConnectTimeout(300000);
               httpUrlConn.setDoOutput(true); //允许写出
               httpUrlConn.setDoInput(true);//允许写入
               httpUrlConn.setUseCaches(false);//不使用缓存

               // 设置请求方式(GET/POST)
               httpUrlConn.setRequestMethod(requestMethod);
               if ("GET".equalsIgnoreCase(requestMethod)) {
                  httpUrlConn.connect();
               }
               // 将返回的输入流转换成字符串
               ips = httpUrlConn.getInputStream();
               out = new BufferedOutputStream(response.getOutputStream());
               int length = 1024;
               int readLength=0;
               byte buf[] = new byte[1024];
               readLength = ips.read(buf, 0, length);
               boolean bool = true;
               while (readLength != -1) {
                  if (bool && new String(buf, "UTF-8").contains("未找到")) {
                     logger.info("==================putIOToResponseByHttp错误信息:"+new String(buf, "UTF-8"));
                     return false;
                  }else{
                     bool = false;
                     out.write(buf, 0, readLength);
                     readLength = ips.read(buf, 0, length);
                  }
               }
            }
            return true;
      }catch (IOException e) {
         errorFilePath(response,"无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }finally {
         try {
            out.flush();
         } catch (IOException e) {

         }
         try {
            out.close();
         } catch (IOException e) {

         }
         try {
            ips.close();
         } catch (IOException e) {

         }
      }
      return false;
   }
   public static InputStream returnIoByHttUrl(String requestMethod,String urlStr){
      try {
         URL url = new URL(urlStr);
         HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
         httpUrlConn.addRequestProperty("Content-type", "application/json");
         //设置请求超时时间
         httpUrlConn.setReadTimeout(30000);
         httpUrlConn.setConnectTimeout(30000);
         httpUrlConn.setDoOutput(true); //允许写出
         httpUrlConn.setDoInput(true);//允许写入
         httpUrlConn.setUseCaches(false);//不使用缓存

         // 设置请求方式(GET/POST)
         httpUrlConn.setRequestMethod(requestMethod);
         if ("GET".equalsIgnoreCase(requestMethod)) {
            httpUrlConn.connect();
         }
         // 将返回的输入流转换成字符串
         return httpUrlConn.getInputStream();
      }catch (IOException e) {
         logger.info("无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }
      return null;
   }


   /***
    * 校验远程地址是否可用
    * @param requestUrl 请求地址
    * @return 是否可用
    */
   public static Boolean checkIP(String requestUrl){
      try {
         String hostAddress = InetAddress.getLocalHost().getHostAddress();
         logger.info("java获取到的当前页面地址"+hostAddress);
         URL url = new URL(requestUrl);
         HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
         httpUrlConn.addRequestProperty("Content-type", "application/json");
         //设置请求超时时间
         httpUrlConn.setReadTimeout(5000);
         httpUrlConn.setConnectTimeout(5000);
         httpUrlConn.setDoOutput(true); //允许写出
         httpUrlConn.setDoInput(true);//允许写入
         httpUrlConn.setUseCaches(false);//不使用缓存
         // 设置请求方式(GET/POST)
         httpUrlConn.setRequestMethod("GET");
         httpUrlConn.connect();
         int responseCode = httpUrlConn.getResponseCode();
         if(responseCode==200){
            logger.info("=============="+requestUrl+":地址有效");
            return true;
         }
      }catch (Exception e){
         return false;
      }
      return false;
   }


   /***
    * 校验远程文件是否可用
    * @return 是否可用
    */
   public static Boolean checkFile(String urlStr){
      InputStream ips = null;
      String requestMethod="GET";
      try {
         URL url = new URL(urlStr);
         HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
         httpUrlConn.addRequestProperty("Content-type", "application/json");
         //httpUrlConn.addRequestProperty("accessToken",accessToken);
         //设置请求超时时间
         httpUrlConn.setReadTimeout(30000);
         httpUrlConn.setConnectTimeout(30000);
         httpUrlConn.setDoOutput(true); //允许写出
         httpUrlConn.setDoInput(true);//允许写入
         httpUrlConn.setUseCaches(false);//不使用缓存

         // 设置请求方式(GET/POST)
         httpUrlConn.setRequestMethod(requestMethod);
         if ("GET".equalsIgnoreCase(requestMethod)) {
            httpUrlConn.connect();
         }
         // 将返回的输入流转换成字符串
         ips = httpUrlConn.getInputStream();
         int length = 1024;
         int readLength=0;
         byte buf[] = new byte[1024];
         readLength = ips.read(buf, 0, length);
         while (readLength != -1) {
            String s = new String(buf, "UTF-8");
            if (s.contains("未找到")) {
               return false;
            }else{
               return true;
            }
         }
      }catch (IOException e) {
         logger.info("无此文件");
      }catch (Exception e) {
         e.printStackTrace();
      }finally {
         try {
            ips.close();
         } catch (IOException e) {

         }
      }
      return false;
   }



}
 

用于单线程下载遇到大文件时读写过于缓慢,查了半天的百度收罗了很多知识,最后整理成一个较为完整的多线程下载工具包

总共三个文件:

One:DownloadFileWithThreadPool

package cn.zectec.hamster.videorecord.download;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class DownloadFileWithThreadPool {

    private static Logger logger = LoggerFactory.getLogger(DownloadFileWithThreadPool.class);

    public void getFileWithThreadPoolByHttp(String urlLocation, String filePath, int poolLength) throws IOException {
        //有顺序的线程
        ExecutorService threadPool = Executors.newSingleThreadExecutor();

        long len = getContentLength(urlLocation);
        for (int i = 0; i < poolLength; i++) {
            long start = i * len / poolLength;
            long end = (i + 1) * len / poolLength - 1;
            if (i == poolLength - 1) {
                end = len;
            }
            DownloadWithRangeByHttpUrl download = new DownloadWithRangeByHttpUrl(urlLocation, filePath, start, end);
            threadPool.execute(download);
        }
        threadPool.shutdown();
    }

    public void getFileWithThreadPoolByLocal(String localAddress, String filePath, int poolLength) throws IOException {
        //有顺序的线程
        ExecutorService threadPool = Executors.newSingleThreadExecutor();
        File file = new File(localAddress);
        if(!file.exists() || !file.isFile()){
            logger.info("多线程下载本地文件:文件不存在");
            return;
        }
        long len = new File(localAddress).length();
        for (int i = 0; i < poolLength; i++) {
            long start = i * len / poolLength;
            DownloadWithRangeByLocalAddress download = new DownloadWithRangeByLocalAddress(localAddress, filePath, len,i,start);
            threadPool.execute(download);
        }
        threadPool.shutdown();
    }

    public static long getContentLength(String urlLocation) throws IOException {
        URL url = null;
        if (urlLocation != null) {
            url = new URL(urlLocation);
        }
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setReadTimeout(5000);
        conn.setRequestMethod("GET");
        long len = conn.getContentLength();

        return len;
    }

}

Two:DownloadWithRangeByHttpUrl

package cn.zectec.hamster.videorecord.download;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;

/***
 * 请求远程http地址,指定文件地址下载文件
 */
public class DownloadWithRangeByHttpUrl implements Runnable {

    /**远程调用地址*/
    private String urlLocation;

    /**文件保存地址*/
    private String filePath;

    /**下载起始位置*/
    private long start;

    /**下载结束位置*/
    private long end;

    DownloadWithRangeByHttpUrl(String urlLocation, String filePath, long start, long end) {
        this.urlLocation = urlLocation;
        this.filePath = filePath;
        this.start = start;
        this.end = end;
    }

    @Override
    public void run() {
        try {
            HttpURLConnection conn = getHttp();
            conn.setRequestProperty("Range", "bytes=" + start + "-" + end);

            File file = new File(filePath);
            RandomAccessFile out = null;
            if (file != null) {
                out = new RandomAccessFile(file, "rw");
            }
            out.seek(start);
            InputStream in = conn.getInputStream();
            byte[] b = new byte[1024];
            int len = 0;
            while ((len = in.read(b)) >= 0) {
                out.write(b, 0, len);
            }
            in.close();
            out.close();
        } catch (Exception e) {
            e.getMessage();
        }

    }

    public HttpURLConnection getHttp() throws IOException {
        URL url = null;
        if (urlLocation != null) {
            url = new URL(urlLocation);
        }
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        //设置请求超时时间
        conn.setReadTimeout(30000);
        conn.setConnectTimeout(30000);
        conn.setDoOutput(true); //允许写出
        conn.setDoInput(true);//允许写入
        conn.setUseCaches(false);//不使用缓存
        conn.setRequestMethod("GET");

        return conn;
    }

}

Three:DownloadWithRangeByLocalAddress

package cn.zectec.hamster.videorecord.download;

import sun.misc.Cleaner;
import sun.nio.ch.DirectBuffer;

import java.io.File;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;

/***
 * 获取本地资源路径,指定文件地址保存文件
 */
public class DownloadWithRangeByLocalAddress implements Runnable {

    // 缓冲区大小为3M
    final int BUFFER_SIZE = 0x300000;

    /**本地文件地址*/
    private String localAddress;

    /**文件保存地址*/
    private String filePath;

    /**如index/10,文件拆分为十段,index为第几段*/
    private long index;

    /**下载起始位置*/
    private long start;

    /**文件大小*/
    private long len;

    DownloadWithRangeByLocalAddress(String localAddress, String filePath, long len,long index,long start) {
        this.localAddress = localAddress;
        this.filePath = filePath;
        this.index = index;
        this.start = start;
        this.len = len;
    }

    @Override
    public void run() {
        try {
            File sourcefile = new File(localAddress);
            RandomAccessFile randomAccessFile = new RandomAccessFile(sourcefile, "r");
            FileChannel channel = randomAccessFile.getChannel();
            MappedByteBuffer inputBuffer = channel.map(FileChannel.MapMode.READ_ONLY, len*index/10,len/10);

            File file = new File(filePath);
            RandomAccessFile out = null;
            if (file != null) {
                out = new RandomAccessFile(file, "rw");
            }
            out.seek(start);
            byte[] dst = new byte[BUFFER_SIZE];// 每次读出3M的内容

            for (int offset = 0; offset < inputBuffer.capacity(); offset += BUFFER_SIZE) {
                if (inputBuffer.capacity() - offset >= BUFFER_SIZE) {

                    for (int i = 0; i < BUFFER_SIZE; i++) {
                        dst[i] = inputBuffer.get(offset + i);
                    }

                } else {

                    for (int i = 0; i < inputBuffer.capacity() - offset; i++) {
                        dst[i] = inputBuffer.get(offset + i);
                    }

                }
                out.write(dst, 0, BUFFER_SIZE);
            }
            out.close();
            inputBuffer.force();
            channel.force(true);
            channel.close();
            randomAccessFile.close();
            unmap(inputBuffer);
        } catch (Exception e) {
            e.getMessage();
        }

    }
    private void unmap(MappedByteBuffer var0) {
        Cleaner var1 = ((DirectBuffer)var0).cleaner();
        if (var1 != null) {
            var1.clean();
        }
    }
}

解说:

 DownloadFileWithThreadPool 该类用于创建线程池并执行相应的任务

里面有两个方法 通过调用这两个方法 来实现多线程下载 远程文件 或者本地文件到指定文件夹下

getFileWithThreadPoolByHttp(String urlLocation, String filePath, int poolLength)

这个方法根据参数说明:

urlLocation 视频远程访问地址

filePath 文件保存的绝对路径

poolLength 线程数

getFileWithThreadPoolByLocal(String localAddress, String filePath, int poolLength)

这个方法根据参数说明:

localAddress 本地文件路径

filePath 文件保存的绝对路径

poolLength 线程数

 

使用:

在最后我只写一下我是怎么使用的,其实调用非常简单只要传对参数就行了

远程下载请求示例:

long time = System.currentTimeMillis();
DownloadFileWithThreadPool pool = new DownloadFileWithThreadPool();
try {
    pool.getFileWithThreadPoolByHttp(urlLocation, filePath, 10);
    System.out.println("下载成功:"+( System.currentTimeMillis() - time )+"毫秒");
} catch (IOException e) {
    e.printStackTrace();
    System.out.println("服务器出错");
}

下载本地文件请求示例

long time = System.currentTimeMillis();
DownloadFileWithThreadPool pool = new DownloadFileWithThreadPool();
try {
    pool.getFileWithThreadPoolByLocal(urlLocation, filePath, 10);
    System.out.println("下载成功:"+( System.currentTimeMillis() - time )+"毫秒");
} catch (IOException e) {
    e.printStackTrace();
    System.out.println("服务器出错");
}

你可能感兴趣的:(多线程下载,io,http,多线程,java)