Java实现多线程下载文件
原文地址：https://blog.dubby.cn/detail.html?id=9090
1.基本原理
先使用head方法查询得到对应文件的Content-Length，然后拆分成多个部分，交由多个线程去处理，使用"Range", "bytes=" + start + "-" + end这个header来指定下载文件的哪个部分。
2.代码实现
为了方便展示，我是用了一个类来实现，其余都是内部类
/**
* Created by yangzheng03 on 2018/1/16.
* https://www.dubby.cn/
*/
public class DownloadTool {
private static String prefix = String.valueOf(System.currentTimeMillis());
private static String path = "";
public static void main(String[] args) {
long startTimestamp = System.currentTimeMillis();
if (args == null || args.length < 1) {
System.out.println("please input the file url.");
return;
}
final String urlString = args[0];
if (args.length >= 2) {
path = args[1];
}
int number = 10;
if (args.length >= 3) {
number = Integer.parseInt(args[2]);
}
System.out.println("Download started, url is \"" + urlString + "\"");
ExecutorService threadPool = Executors.newFixedThreadPool(number);
try {
URL url = new URL(urlString);
HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod("HEAD");
String contentLengthStr = httpURLConnection.getHeaderField("Content-Length");
long contentLength = Long.parseLong(contentLengthStr);
if (contentLength > 1024 * 1024) {
System.out.println("Content-Length\t" + (contentLength / 1024 / 1024) + "MB");
} else if (contentLength > 1024) {
System.out.println("Content-Length\t" + (contentLength / 1024) + "KB");
} else {
System.out.println("Content-Length\t" + (contentLength) + "B");
}
long tempLength = contentLength / number;
long start = 0, end = -1;
Map> futureMap = new HashMap<>(number);
for (int i = 0; i < number; ++i) {
start = end + 1;
end = end + tempLength;
if (i == number - 1) {
end = contentLength;
}
System.out.println("start:\t" + start + "\tend:\t" + end);
DownloadThread thread = new DownloadThread(i, start, end, urlString);
futureMap.put(i, threadPool.submit(thread));
}
System.out.println();
String filename = urlString.substring(urlString.lastIndexOf("/") + 1);
if (!path.equals("")) {
filename = path + "/" + filename;
}
RandomAccessFile resultFile = new RandomAccessFile(filename, "rw");
for (int i = 0; i < number; ++i) {
Future future = futureMap.get(i);
DownloadTemp temp = future.get();
RandomAccessFile tempFile = new RandomAccessFile(temp.filename, "r");
tempFile.getChannel().transferTo(0, tempFile.length(), resultFile.getChannel());
}
threadPool.shutdown();
} catch (Exception e) {
e.printStackTrace();
}
long completedTimestamp = System.currentTimeMillis();
System.out.println();
System.out.println("cost " + (completedTimestamp - startTimestamp) / 1000 + "s");
}
private static class DownloadThread implements Callable {
private int index;
private String filename;
private long start, end;
private String urlString;
DownloadThread(int index, long start, long end, String url) {
this.urlString = url;
this.index = index;
this.start = start;
this.end = end;
if (path.equals("")) {
this.filename = prefix + "-temp-" + index;
} else {
this.filename = path + "/" + prefix + "-temp-" + index;
}
}
@Override
public DownloadTemp call() throws Exception {
URL url = new URL(urlString);
HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setRequestProperty("Range", "bytes=" + start + "-" + end);
InputStream inputStream = null;
OutputStream outputStream = null;
try {
inputStream = httpURLConnection.getInputStream();
File file = new File(filename);
outputStream = new FileOutputStream(file);
while (true) {
byte[] bytes = new byte[10240];
int length = inputStream.read(bytes);
if (length <= 0) {
break;
}
outputStream.write(bytes, 0, length);
}
outputStream.flush();
} finally {
if (inputStream != null) {
inputStream.close();
}
if (outputStream != null) {
outputStream.close();
}
}
DownloadTemp downloadTemp = new DownloadTemp();
downloadTemp.index = index;
downloadTemp.filename = filename;
System.out.println("thread\t" + index + "\tcompleted.");
return downloadTemp;
}
}
private static class DownloadTemp {
private int index;
private String filename;
}
} 3.使用Jar包
下载工具 download-1.0.jar
下载下来后，改名为download-1.0.jar，执行：
java -jar download-1.0.jar https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip /Users/test/Desktop/download 15其中有三个参数:
- 下载的URL，必填
- 下载到目标目录，选填，如果不指定，下载到当前目前下
- 开启的线程数，选填，默认是10个线程
测试结果:
➜ target java -jar download-1.0.jar https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip /Users/test/Desktop/download 15
Download started, url is "https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip"
Content-Length 1MB
start: 0 end: 83780
start: 83781 end: 167561
start: 167562 end: 251342
start: 251343 end: 335123
start: 335124 end: 418904
start: 418905 end: 502685
start: 502686 end: 586466
start: 586467 end: 670247
start: 670248 end: 754028
start: 754029 end: 837809
start: 837810 end: 921590
start: 921591 end: 1005371
start: 1005372 end: 1089152
start: 1089153 end: 1172933
start: 1172934 end: 1256726
thread 0 completed.
thread 10 completed.
thread 1 completed.
thread 9 completed.
thread 14 completed.
thread 11 completed.
thread 7 completed.
thread 6 completed.
thread 2 completed.
thread 4 completed.
thread 3 completed.
thread 8 completed.
thread 12 completed.
thread 13 completed.
thread 5 completed.
cost 9s