java 多线程下载文件
关于下载文件，首先需要了解一点io和http请求，线程等相关知识，然后才能一步一步的趴坑整理出一个较为满意的结果。原先我写过一个较为简单的下载是单线程的，如下
package cn.zectec.hamster.baseservice.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.*;
import java.util.HashMap;
import java.util.Map;
public class DownloadUtil {
private static Logger logger = LoggerFactory.getLogger(DownloadUtil.class);
public static void errorFilePath(HttpServletResponse response, String msg){
//没有该消息记录直接返回无此文件
try {
ServletOutputStream out=response.getOutputStream();
OutputStreamWriter ow=new OutputStreamWriter(out,"UTF-8");
ow.write(msg);
ow.flush();
ow.close();
} catch (IOException e) {
}catch (Exception e) {
e.printStackTrace();
}
}
/**
* @param filePath 要下载的文件路径
* @param returnName 返回的文件名
* @param response HttpServletResponse
* @param delFlag 是否删除文件
*/
protected void download(String filePath,String returnName,HttpServletResponse response,boolean delFlag){
prototypeDownload(new File(filePath), returnName, response, delFlag);
}
/**
* @param file 要下载的文件
* @param returnName 返回的文件名
* @param response HttpServletResponse
* @param delFlag 是否删除文件
*/
public static void download(File file, String returnName, HttpServletResponse response, boolean delFlag){
prototypeDownload(file, returnName, response, delFlag);
}
/**
* @param file 要下载的文件
* @param returnName 返回的文件名
* @param response HttpServletResponse
* @param delFlag 是否删除文件
*/
private static void prototypeDownload(File file, String returnName, HttpServletResponse response, boolean delFlag){
// 下载文件
FileInputStream inputStream = null;
ServletOutputStream outputStream = null;
try {
if(!file.exists()) return;
response.reset();
//设置响应类型 PDF文件为"application/pdf"，WORD文件为："application/msword"， EXCEL文件为："application/vnd.ms-excel"。
response.setContentType("application/octet-stream;charset=utf-8");
//设置响应的文件名称,并转换成中文编码
returnName = URLEncoder.encode(returnName,"UTF-8");
returnName = response.encodeURL(new String(returnName.getBytes(),"iso8859-1")); //保存的文件名,必须和页面编码一致,否则乱码
//attachment作为附件下载；inline客户端机器有安装匹配程序，则直接打开；注意改变配置，清除缓存，否则可能不能看到效果
response.addHeader("Content-Disposition", "attachment;filename="+returnName);
//将文件读入响应流
inputStream = new FileInputStream(file);
outputStream = response.getOutputStream();
int length = 1024;
int readLength=0;
byte buf[] = new byte[1024];
readLength = inputStream.read(buf, 0, length);
while (readLength != -1) {
outputStream.write(buf, 0, readLength);
readLength = inputStream.read(buf, 0, length);
}
}catch (IOException e) {
errorFilePath(response,"无此文件");
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
outputStream.flush();
} catch (IOException e) {
}
try {
outputStream.close();
} catch (IOException e) {
}
try {
inputStream.close();
} catch (IOException e) {
}
//删除原文件
if(delFlag) {
file.delete();
}
}
}
/**
* by tony 2013-10-17
* @param byteArrayOutputStream 将文件内容写入ByteArrayOutputStream
* @param response HttpServletResponse 写入response
* @param returnName 返回的文件名
*/
public void download(ByteArrayOutputStream byteArrayOutputStream, HttpServletResponse response, String returnName) throws IOException{
response.setContentType("application/octet-stream;charset=utf-8");
returnName = response.encodeURL(new String(returnName.getBytes(),"iso8859-1")); //保存的文件名,必须和页面编码一致,否则乱码
response.addHeader("Content-Disposition", "attachment;filename=" + returnName);
response.setContentLength(byteArrayOutputStream.size());
ServletOutputStream outputstream = response.getOutputStream(); //取得输出流
byteArrayOutputStream.writeTo(outputstream); //写到输出流
byteArrayOutputStream.close(); //关闭
outputstream.flush(); //刷数据
}
/***
* 浏览器远程下载文件
*/
public static void downloadFileByHttpUrl(HttpServletResponse response,String urlStr,String fileName){
OutputStream out = null;
InputStream ips = null;
try {
logger.info("第一次请求url==>" + urlStr);
URL url = new URL(urlStr);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//设置请求超时时间
httpUrlConn.setReadTimeout(15000);
httpUrlConn.setConnectTimeout(15000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod("GET");
httpUrlConn.connect();
int responseCode = httpUrlConn.getResponseCode();
if(responseCode!=200){
errorFilePath(response,"无此文件");
}else {
ips = httpUrlConn.getInputStream();//字节流 输入流
response.reset();
response.setContentType("application/x-download");
response.addHeader("Content-Disposition","attachment;filename="+ new String(fileName.getBytes(),"iso-8859-1"));
response.setContentType("application/octet-stream");
out = new BufferedOutputStream(response.getOutputStream());
int length = 1024;
int readLength=0;
byte buf[] = new byte[1024];
readLength = ips.read(buf, 0, length);
while (readLength != -1) {
out.write(buf, 0, readLength);
readLength = ips.read(buf, 0, length);
}
}
}catch (IOException e) {
errorFilePath(response,"无此文件");
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
out.flush();
} catch (IOException e) {
}
try {
out.close();
} catch (IOException e) {
}
try {
ips.close();
} catch (IOException e) {
}
}
}
/***
* http 请求获取下载文件保存到本地
* urlStr 远程地址
* filePath 指定本地保存路径
*/
public static Map downloadFileByHttpUrl(String urlStr, String filePath){
Map map = new HashMap<>();
map.put("success",false);
OutputStream out = null;
InputStream ips = null;
String requestMethod="GET";
try {
File file = new File(filePath);
logger.info("===================："+urlStr);
logger.info("===================获取流媒体服务中的文件："+filePath);
URL url = new URL(urlStr);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//httpUrlConn.addRequestProperty("accessToken",accessToken);
//设置请求超时时间
httpUrlConn.setReadTimeout(30000);
httpUrlConn.setConnectTimeout(30000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod(requestMethod);
if ("GET".equalsIgnoreCase(requestMethod)) {
httpUrlConn.connect();
}
// 将返回的输入流转换成字符串
ips = httpUrlConn.getInputStream();
out = new FileOutputStream(file,false);
int length = 1024;
int readLength=0;
byte buf[] = new byte[1024];
readLength = ips.read(buf, 0, length);
while (readLength != -1) {
String s = new String(buf, "UTF-8");
if (s.contains("未找到")) {
logger.info("==================putIOToResponseByHttp错误信息："+s);
file.delete();
map.put("msg",s);
return map;
}else{
out.write(buf, 0, readLength);
readLength = ips.read(buf, 0, length);
}
}
map.put("success",true);
}catch (IOException e) {
logger.info("无此文件");
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
out.flush();
} catch (IOException e) {
}
try {
out.close();
} catch (IOException e) {
}
try {
ips.close();
} catch (IOException e) {
}
}
return map;
}
/***
* 根据远程http地址将流存到respons 返回失败或者成功
* @param response 浏览器response
* @param urlStr 远程http地址
* @return 返回失败或者成功
*/
public static boolean putIOToResponseByHttp(HttpServletResponse response, String urlStr) {
OutputStream out = null;
InputStream ips = null;
String requestMethod="GET";
try {
if (StringUtils.isEmpty(urlStr)) {
return false;
}else{
URL url = new URL(urlStr);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//httpUrlConn.addRequestProperty("accessToken",accessToken);
//设置请求超时时间
httpUrlConn.setReadTimeout(300000);
httpUrlConn.setConnectTimeout(300000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod(requestMethod);
if ("GET".equalsIgnoreCase(requestMethod)) {
httpUrlConn.connect();
}
// 将返回的输入流转换成字符串
ips = httpUrlConn.getInputStream();
out = new BufferedOutputStream(response.getOutputStream());
int length = 1024;
int readLength=0;
byte buf[] = new byte[1024];
readLength = ips.read(buf, 0, length);
boolean bool = true;
while (readLength != -1) {
if (bool && new String(buf, "UTF-8").contains("未找到")) {
logger.info("==================putIOToResponseByHttp错误信息："+new String(buf, "UTF-8"));
return false;
}else{
bool = false;
out.write(buf, 0, readLength);
readLength = ips.read(buf, 0, length);
}
}
}
return true;
}catch (IOException e) {
errorFilePath(response,"无此文件");
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
out.flush();
} catch (IOException e) {
}
try {
out.close();
} catch (IOException e) {
}
try {
ips.close();
} catch (IOException e) {
}
}
return false;
}
public static InputStream returnIoByHttUrl(String requestMethod,String urlStr){
try {
URL url = new URL(urlStr);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//设置请求超时时间
httpUrlConn.setReadTimeout(30000);
httpUrlConn.setConnectTimeout(30000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod(requestMethod);
if ("GET".equalsIgnoreCase(requestMethod)) {
httpUrlConn.connect();
}
// 将返回的输入流转换成字符串
return httpUrlConn.getInputStream();
}catch (IOException e) {
logger.info("无此文件");
}catch (Exception e) {
e.printStackTrace();
}
return null;
}
/***
* 校验远程地址是否可用
* @param requestUrl 请求地址
* @return 是否可用
*/
public static Boolean checkIP(String requestUrl){
try {
String hostAddress = InetAddress.getLocalHost().getHostAddress();
logger.info("java获取到的当前页面地址"+hostAddress);
URL url = new URL(requestUrl);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//设置请求超时时间
httpUrlConn.setReadTimeout(5000);
httpUrlConn.setConnectTimeout(5000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod("GET");
httpUrlConn.connect();
int responseCode = httpUrlConn.getResponseCode();
if(responseCode==200){
logger.info("=============="+requestUrl+":地址有效");
return true;
}
}catch (Exception e){
return false;
}
return false;
}
/***
* 校验远程文件是否可用
* @return 是否可用
*/
public static Boolean checkFile(String urlStr){
InputStream ips = null;
String requestMethod="GET";
try {
URL url = new URL(urlStr);
HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
httpUrlConn.addRequestProperty("Content-type", "application/json");
//httpUrlConn.addRequestProperty("accessToken",accessToken);
//设置请求超时时间
httpUrlConn.setReadTimeout(30000);
httpUrlConn.setConnectTimeout(30000);
httpUrlConn.setDoOutput(true); //允许写出
httpUrlConn.setDoInput(true);//允许写入
httpUrlConn.setUseCaches(false);//不使用缓存
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod(requestMethod);
if ("GET".equalsIgnoreCase(requestMethod)) {
httpUrlConn.connect();
}
// 将返回的输入流转换成字符串
ips = httpUrlConn.getInputStream();
int length = 1024;
int readLength=0;
byte buf[] = new byte[1024];
readLength = ips.read(buf, 0, length);
while (readLength != -1) {
String s = new String(buf, "UTF-8");
if (s.contains("未找到")) {
return false;
}else{
return true;
}
}
}catch (IOException e) {
logger.info("无此文件");
}catch (Exception e) {
e.printStackTrace();
}finally {
try {
ips.close();
} catch (IOException e) {
}
}
return false;
}
}
用于单线程下载遇到大文件时读写过于缓慢，查了半天的百度收罗了很多知识，最后整理成一个较为完整的多线程下载工具包
总共三个文件：
One：DownloadFileWithThreadPool
package cn.zectec.hamster.videorecord.download;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class DownloadFileWithThreadPool {
private static Logger logger = LoggerFactory.getLogger(DownloadFileWithThreadPool.class);
public void getFileWithThreadPoolByHttp(String urlLocation, String filePath, int poolLength) throws IOException {
//有顺序的线程
ExecutorService threadPool = Executors.newSingleThreadExecutor();
long len = getContentLength(urlLocation);
for (int i = 0; i < poolLength; i++) {
long start = i * len / poolLength;
long end = (i + 1) * len / poolLength - 1;
if (i == poolLength - 1) {
end = len;
}
DownloadWithRangeByHttpUrl download = new DownloadWithRangeByHttpUrl(urlLocation, filePath, start, end);
threadPool.execute(download);
}
threadPool.shutdown();
}
public void getFileWithThreadPoolByLocal(String localAddress, String filePath, int poolLength) throws IOException {
//有顺序的线程
ExecutorService threadPool = Executors.newSingleThreadExecutor();
File file = new File(localAddress);
if(!file.exists() || !file.isFile()){
logger.info("多线程下载本地文件：文件不存在");
return;
}
long len = new File(localAddress).length();
for (int i = 0; i < poolLength; i++) {
long start = i * len / poolLength;
DownloadWithRangeByLocalAddress download = new DownloadWithRangeByLocalAddress(localAddress, filePath, len,i,start);
threadPool.execute(download);
}
threadPool.shutdown();
}
public static long getContentLength(String urlLocation) throws IOException {
URL url = null;
if (urlLocation != null) {
url = new URL(urlLocation);
}
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setReadTimeout(5000);
conn.setRequestMethod("GET");
long len = conn.getContentLength();
return len;
}
}Two:DownloadWithRangeByHttpUrl
package cn.zectec.hamster.videorecord.download;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
/***
* 请求远程http地址，指定文件地址下载文件
*/
public class DownloadWithRangeByHttpUrl implements Runnable {
/**远程调用地址*/
private String urlLocation;
/**文件保存地址*/
private String filePath;
/**下载起始位置*/
private long start;
/**下载结束位置*/
private long end;
DownloadWithRangeByHttpUrl(String urlLocation, String filePath, long start, long end) {
this.urlLocation = urlLocation;
this.filePath = filePath;
this.start = start;
this.end = end;
}
@Override
public void run() {
try {
HttpURLConnection conn = getHttp();
conn.setRequestProperty("Range", "bytes=" + start + "-" + end);
File file = new File(filePath);
RandomAccessFile out = null;
if (file != null) {
out = new RandomAccessFile(file, "rw");
}
out.seek(start);
InputStream in = conn.getInputStream();
byte[] b = new byte[1024];
int len = 0;
while ((len = in.read(b)) >= 0) {
out.write(b, 0, len);
}
in.close();
out.close();
} catch (Exception e) {
e.getMessage();
}
}
public HttpURLConnection getHttp() throws IOException {
URL url = null;
if (urlLocation != null) {
url = new URL(urlLocation);
}
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
//设置请求超时时间
conn.setReadTimeout(30000);
conn.setConnectTimeout(30000);
conn.setDoOutput(true); //允许写出
conn.setDoInput(true);//允许写入
conn.setUseCaches(false);//不使用缓存
conn.setRequestMethod("GET");
return conn;
}
}
Three:DownloadWithRangeByLocalAddress
package cn.zectec.hamster.videorecord.download;
import sun.misc.Cleaner;
import sun.nio.ch.DirectBuffer;
import java.io.File;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
/***
* 获取本地资源路径，指定文件地址保存文件
*/
public class DownloadWithRangeByLocalAddress implements Runnable {
// 缓冲区大小为3M
final int BUFFER_SIZE = 0x300000;
/**本地文件地址*/
private String localAddress;
/**文件保存地址*/
private String filePath;
/**如index/10,文件拆分为十段，index为第几段*/
private long index;
/**下载起始位置*/
private long start;
/**文件大小*/
private long len;
DownloadWithRangeByLocalAddress(String localAddress, String filePath, long len,long index,long start) {
this.localAddress = localAddress;
this.filePath = filePath;
this.index = index;
this.start = start;
this.len = len;
}
@Override
public void run() {
try {
File sourcefile = new File(localAddress);
RandomAccessFile randomAccessFile = new RandomAccessFile(sourcefile, "r");
FileChannel channel = randomAccessFile.getChannel();
MappedByteBuffer inputBuffer = channel.map(FileChannel.MapMode.READ_ONLY, len*index/10,len/10);
File file = new File(filePath);
RandomAccessFile out = null;
if (file != null) {
out = new RandomAccessFile(file, "rw");
}
out.seek(start);
byte[] dst = new byte[BUFFER_SIZE];// 每次读出3M的内容
for (int offset = 0; offset < inputBuffer.capacity(); offset += BUFFER_SIZE) {
if (inputBuffer.capacity() - offset >= BUFFER_SIZE) {
for (int i = 0; i < BUFFER_SIZE; i++) {
dst[i] = inputBuffer.get(offset + i);
}
} else {
for (int i = 0; i < inputBuffer.capacity() - offset; i++) {
dst[i] = inputBuffer.get(offset + i);
}
}
out.write(dst, 0, BUFFER_SIZE);
}
out.close();
inputBuffer.force();
channel.force(true);
channel.close();
randomAccessFile.close();
unmap(inputBuffer);
} catch (Exception e) {
e.getMessage();
}
}
private void unmap(MappedByteBuffer var0) {
Cleaner var1 = ((DirectBuffer)var0).cleaner();
if (var1 != null) {
var1.clean();
}
}
}解说：
DownloadFileWithThreadPool 该类用于创建线程池并执行相应的任务
里面有两个方法 通过调用这两个方法 来实现多线程下载 远程文件 或者本地文件到指定文件夹下
getFileWithThreadPoolByHttp(String urlLocation, String filePath, int poolLength)这个方法根据参数说明：
urlLocation 视频远程访问地址
filePath 文件保存的绝对路径
poolLength 线程数
getFileWithThreadPoolByLocal(String localAddress, String filePath, int poolLength)这个方法根据参数说明：
localAddress 本地文件路径
filePath 文件保存的绝对路径
poolLength 线程数
使用：
在最后我只写一下我是怎么使用的，其实调用非常简单只要传对参数就行了
远程下载请求示例：
long time = System.currentTimeMillis();
DownloadFileWithThreadPool pool = new DownloadFileWithThreadPool();
try {
pool.getFileWithThreadPoolByHttp(urlLocation, filePath, 10);
System.out.println("下载成功："+( System.currentTimeMillis() - time )+"毫秒");
} catch (IOException e) {
e.printStackTrace();
System.out.println("服务器出错");
}下载本地文件请求示例
long time = System.currentTimeMillis();
DownloadFileWithThreadPool pool = new DownloadFileWithThreadPool();
try {
pool.getFileWithThreadPoolByLocal(urlLocation, filePath, 10);
System.out.println("下载成功："+( System.currentTimeMillis() - time )+"毫秒");
} catch (IOException e) {
e.printStackTrace();
System.out.println("服务器出错");
}