java多线程批量下载文件打包成zip
本文叙述
注: 本文中使用到的PdfUtil工具类暂不提供自行剔除
本文用于多个文件批量下载，提供两种下载方式 （二者利弊自行考量）
1. 将文件文件全部下载到本地文件夹，之后将文件夹打包成zip最后输出到浏览器再删除文件夹跟zip文件利用本地磁盘作为过渡
2. 直接在代码中拿到全部文件的byte[]数组之后，将文件的byte[]数组全部装入zip文件流，最后将zip文件流输出到浏览器，全过程不产生实体文件到磁盘中。
前端调用方式: 前端调用接口方式本文是直接通过 window.location.href 方式调用接口地址 或通过以下js方式调用本文不做叙述自行了解其他方式以下提供案例代码
js文件
import axios from 'axios'
import { getToken } from '@/utils/auth'
const mimeMap = {
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
zip: 'application/zip'
}
const baseUrl = process.env.VUE_APP_BASE_API //接口前缀换成自己的
export function downLoadZip(str, filename) {
var url = baseUrl + str
axios({
method: 'get',
url: url,
responseType: 'blob',
headers: { 'Authorization': 'Bearer ' + getToken() }
}).then(res => {
resolveBlob(res, mimeMap.zip)
})
}
/**
* 解析blob响应内容并下载
* @param {*} res blob响应内容
* @param {String} mimeType MIME类型
*/
export function resolveBlob(res, mimeType) {
const aLink = document.createElement('a')
var blob = new Blob([res.data], { type: mimeType })
// //从response的headers中获取filename, 后端response.setHeader("Content-disposition", "attachment; filename=xxxx.docx") 设置的文件名;
var patt = new RegExp('filename=([^;]+\\.[^\\.;]+);*')
var contentDisposition = decodeURI(res.headers['content-disposition'])
var result = patt.exec(contentDisposition)
var fileName = result[1]
fileName = fileName.replace(/\"/g, '')
aLink.href = URL.createObjectURL(blob)
aLink.setAttribute('download', fileName) // 设置下载文件名称
document.body.appendChild(aLink)
aLink.click()
document.body.appendChild(aLink)
}
在具体页面模块引入该文件的downLoadZip 方法
import { downLoadZip } from "@/utils/zipdownload";
在具体事件上调用下载方法
handleGenTable(row) {
downLoadZip("传入接口路径从controller层开始填写" , "zip文件名称");// 例如downLoadZip("/test/downLoadZip", "test");
}
以下附上后端代码
下载工具类
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.OSSObject;
import com.nuoer.common.utils.aliyun.CloudConstant;
import com.nuoer.common.utils.pdf.PdfUtil;
import com.nuoer.project.cv.domain.SkillCert;
import com.nuoer.project.personalCenter.domain.MineDto;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
/**
* 文件下载类
* 本类提供两种多线程批量下载方式
* 方式一 : 实体文件储存到本地磁盘后输出到浏览器
* 方式二: 直接处理远程拉取到的文件流byte[] 数组后输出到浏览器
*/
public class DownloadUtil {
private static Logger logger = LoggerFactory.getLogger(DownloadUtil.class);
/**
* 下载线程数
*/
private static final int DOWNLOAD_THREAD_NUM = 14;
/**
* 下载线程池
*/
private static ExecutorService downloadExecutorService = ThreadUtil
.buildDownloadBatchThreadPool(DOWNLOAD_THREAD_NUM);
/**
* 创建文件夹,如果文件夹已经存在或者创建成功返回true
*
* @param path
* 路径
* @return boolean
*/
private static boolean createFolderIfNotExists(String path) {
String folderName = getFolder(path);
if (folderName.equals(path)) {
return true;
}
File folder = new File(getFolder(path));
if (!folder.exists()) {
synchronized (DownloadUtil.class) {
if (!folder.exists()) {
return folder.mkdirs();
}
}
}
return true;
}
/**
* 获取文件夹
*
* @param path
* 文件路径
* @return String
*/
private static String getFolder(String path) {
int index = path.lastIndexOf("/");
return -1 != index ? path.substring(0, index) : path;
}
/**
* 获取最后一个元素
*
* @param size
* 列表长度
* @param index
* 下标
* @return int
*/
private static int getLastNum(int size, int index) {
return index > size ? size : index;
}
/**
* 获取划分页面数量
*
* @param size
* 列表长度
* @return int
*/
private static int getPageNum(int size) {
int tmp = size / DOWNLOAD_THREAD_NUM;
return size % DOWNLOAD_THREAD_NUM == 0 ? tmp : tmp + 1;
}
/**
* 通过http请求获取远程文件信息 文件下载
*
* @param fileUrl
* 文件url,如:https://img3.doubanio.com//view//photo//s_ratio_poster//public//p2369390663.webp
* @param path
* 存放路径,如: /opt/img/douban/my.webp
*/
//TODO 实体文件落地方式
public static void downloadHttp(String fileUrl, String path) {
// 判断存储文件夹是否已经存在或者创建成功
if (!createFolderIfNotExists(path)) {
logger.error("We can't create folder:{}", getFolder(path));
return;
}
InputStream in = null;
FileOutputStream out = null;
try {
URL url = new URL(fileUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
// 2s
conn.setConnectTimeout(10000);
in = conn.getInputStream();
out = new FileOutputStream(path);
int len;
byte[] arr = new byte[1024 * 1000];
while (-1 != (len = in.read(arr))) {
out.write(arr, 0, len);
}
out.flush();
conn.disconnect();
} catch (Exception e) {
logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());
} finally {
try {
if (null != out) {
out.close();
}
if (null != in) {
in.close();
}
} catch (Exception e) {
// do nothing
}
}
}
/**
* 通过http请求获取远程文件信息 文件下载
*
* @param fileUrl
* 文件url,如:https://img3.doubanio.com//view//photo//s_ratio_poster//public//p2369390663.webp
* @param path
* 存放路径,如: /opt/img/douban/my.webp
*/
//TODO 实体文件不落地方式
public static void downloadHttpByte(String fileUrl, String path,Map byteArrayMap) throws IOException {
InputStream in = null;
try {
URL url = new URL(fileUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
// 2s
conn.setConnectTimeout(10000);
in = conn.getInputStream();
byteArrayMap.put(path, IOUtils.toByteArray(in));
conn.disconnect();
} catch (Exception e) {
logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());
}finally {
if (null != in ){
in.close();
}
}
}
/**
* 通过阿里云客户端对象获取文件信息
* @param fileUrl
* @param fileName
*/
//TODO 实体文件落地方式
public static void downloadAliyunOss(String fileUrl, String fileName,OSSClient ossClient) {
// 判断存储文件夹是否已经存在或者创建成功
/*if (!createFolderIfNotExists(path)) {
logger.error("We can't create folder:{}", getFolder(path));
return;
}*/
// 读去Object内容 返回
InputStream in = null;
FileOutputStream out = null;
OSSObject ossObject = null;
try {
String url = fileUrl.substring(fileUrl.indexOf(".com/")+".com/".length());
ossObject = ossClient.getObject(CloudConstant.BUCKET, url);
in = ossObject.getObjectContent();
out = new FileOutputStream(fileName);
int len;
byte[] arr = new byte[1024 * 1000];
while (-1 != (len = in.read(arr))) {
out.write(arr, 0, len);
}
out.flush();
} catch (Exception e) {
logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());
} finally {
try {
if (null != out) {
out.close();
}
if (null != in) {
in.close();
}
if (null != ossObject){
ossObject.close();
}
} catch (Exception e) {
// do nothing
}
}
}
/**
* 通过阿里云客户端对象获取文件信息
* @param fileUrl
* @param fileName
*/
//TODO 实体文件不落地方式
public static void downloadAliyunOssByte(String fileUrl, String fileName,OSSClient ossClient,Map map) throws IOException {
// 判断存储文件夹是否已经存在或者创建成功
/*if (!createFolderIfNotExists(path)) {
logger.error("We can't create folder:{}", getFolder(path));
return;
}*/
InputStream inputStream = null;
OSSObject ossObject = null;
try {
String url = fileUrl.substring(fileUrl.indexOf(".com/")+".com/".length());
ossObject = ossClient.getObject(CloudConstant.BUCKET, url);
inputStream = ossObject.getObjectContent();
map.put(fileName, IOUtils.toByteArray(inputStream));
} catch (Exception e) {
logger.error("Fail to download: {} by {}", fileUrl, e.getMessage());
} finally {
if (null != inputStream){
inputStream.close();
}
if (null != ossObject){
ossObject.close();
}
}
}
/**
* 生成zip文件并向浏览器输出文件流
*/
public static void createdWriteZip(HttpServletRequest request, HttpServletResponse response, String zipFileName, byte[] data) throws IOException
{
String header = request.getHeader("User-Agent").toUpperCase();
if (header.contains("MSIE") || header.contains("TRIDENT") || header.contains("EDGE")) {
zipFileName = URLEncoder.encode(zipFileName, "utf-8");
zipFileName = zipFileName.replace("+", "%20"); //IE下载文件名空格变+号问题
} else {
zipFileName = new String(zipFileName.getBytes(), "ISO8859-1");
}
response.reset();
response.setHeader("Content-Disposition", "attachment; filename=" + zipFileName);
response.addHeader("Content-Length", "" + data.length);
response.setContentType("application/octet-stream; charset=UTF-8");
IOUtils.write(data, response.getOutputStream());
}
/**
* 下载资源
*
* issue: 线程池创建过多
*
* 最大批量下载为5,请知悉
*
* @param resourceMap
* 资源map, key为资源下载url,value为资源存储位置
*/
//TODO 实体文件落地方式
public static void batch(Map resourceMap,Map mineDtoMap,File path,OSSClient ossClient,
String resumeFlag,String onlineFlag) {
if (resourceMap == null || resourceMap.isEmpty()) {
return;
}
try {
List keys = new ArrayList<>(resourceMap.keySet());
int size = keys.size();
int pageNum = getPageNum(size);
for (int index = 0; index < pageNum; index++) {
int start = index * DOWNLOAD_THREAD_NUM;
int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);
final CountDownLatch latch = new CountDownLatch(last - start);
// 获取列表子集
List urlList = keys.subList(start, last);
for (String url : urlList) {
// 提交任务
// Runnable task = new DownloadWorker(latch, url, resourceMap.get(url)); 正式使用
Runnable task = url.contains(onlineFlag)?
new CreatedPdfWorker(latch,path, mineDtoMap.get(url.replace(onlineFlag,""))): //测试使用
new DownloadWorker(latch,url.substring(url.indexOf(resumeFlag)+resumeFlag.length()), resourceMap.get(url),ossClient); //测试使用
downloadExecutorService.submit(task);
}
latch.await();
}
} catch (Exception e) {
logger.error("{}", e);
}
logger.info("Download resource map is all done");
}
/**
* 下载资源
*
* issue: 线程池创建过多
*
* 最大批量下载为5,请知悉
* @param resourceMap 资源map, key为资源下载url,value为资源存储位置包含文件类型后缀
* @param mineDtoMap 在线简历信息 key为用户id加batchByte()方法中onlineFlag参数，value为简历信息
* @param ossClient 阿里云oss对象
* @param resumeFlag 附件简历标识
* @param onlineFlag 在线简历标识
* @param byteArrayMap new 传入对象即可
* @return Map 文件字节数组 key为件名称必须包含文件类型后缀，value为文件字节数组
*/
//TODO 实体文件不落地方式
public static Map batchByte(Map resourceMap, Map mineDtoMap, OSSClient ossClient,
String resumeFlag, String onlineFlag,Map byteArrayMap) {
if (resourceMap == null || resourceMap.isEmpty()) {
return null;
}
try {
List keys = new ArrayList<>(resourceMap.keySet());
int size = keys.size();
int pageNum = getPageNum(size);
for (int index = 0; index < pageNum; index++) {
int start = index * DOWNLOAD_THREAD_NUM;
int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);
final CountDownLatch latch = new CountDownLatch(last - start);
// 获取列表子集
List urlList = keys.subList(start, last);
for (String url : urlList) {
// 提交任务
// Runnable task = new DownloadWorker(latch, url, resourceMap.get(url)); 正式使用
Runnable task = url.contains(onlineFlag)?
new CreatedPdfWorkerByte(latch, mineDtoMap.get(url.replace(onlineFlag,"")),byteArrayMap): //测试使用
new DownloadWorkerByte(latch,url.substring(url.indexOf(resumeFlag)+resumeFlag.length()),resourceMap.get(url),ossClient,byteArrayMap); //测试使用
downloadExecutorService.submit(task);
}
latch.await();
}
} catch (Exception e) {
e.printStackTrace();
}
logger.info("Download resource map is all done");
return byteArrayMap;
}
/**
* 下载线程 实体文件落地方式
*/
//TODO 下载线程 实体文件落地方式
static class DownloadWorker implements Runnable {
private CountDownLatch latch;
private OSSClient ossClient;
private String url;
private String fileName;
DownloadWorker(CountDownLatch latch, String url, String fileName,OSSClient ossClient) {
this.latch = latch;
this.url = url;
this.fileName = fileName;
this.ossClient = ossClient;
}
@Override
public void run() {
logger.debug("Start batch:[{}] into: [{}]", url, fileName);
DownloadUtil.downloadAliyunOss(url, fileName,ossClient); //
//DownloadUtil.downloadHttp(url, path);
logger.debug("Download:[{}] into: [{}] is done", url, fileName);
latch.countDown();
}
}
/**
* 生成pdf线程 实体文件落地方式
*/
//TODO 生成pdf线程 实体文件落地方式
static class CreatedPdfWorker implements Runnable {
private CountDownLatch latch;
private MineDto mineDto;
private File path;
CreatedPdfWorker(CountDownLatch latch, File path, MineDto mineDto) {
this.latch = latch;
this.mineDto = mineDto;
this.path = path;
}
@Override
public void run() {
logger.debug("Start batch:[{}] into: [{}]", mineDto.getTbMember().getUsername(), path);
List skillCertList = new ArrayList<>();
skillCertList.add(mineDto.getSkillCert());
PdfUtil.createdPdfDirectory(path,mineDto.getTbMember(),mineDto.getEducationalExp(),mineDto.getSocialPractice(),mineDto.getCampusActivities(),skillCertList);
logger.debug("CreatedPdf:[{}] into: [{}] is done", mineDto.getTbMember().getUsername(), path);
latch.countDown();
}
}
/**
* 下载线程
*/
//TODO 下载线程 文件不落地方式
static class DownloadWorkerByte implements Runnable {
private CountDownLatch latch;
private OSSClient ossClient;
private String url;
private String fileName;
private Map byteArrayMap;
DownloadWorkerByte(CountDownLatch latch, String url, String fileName,OSSClient ossClient,Map byteArrayMap) {
this.latch = latch;
this.url = url;
this.fileName = fileName;
this.ossClient = ossClient;
this.byteArrayMap = byteArrayMap;
}
@SneakyThrows
@Override
public void run() {
logger.debug("Start batch:[{}] into: [{}]", url, fileName);
DownloadUtil.downloadAliyunOssByte(url, fileName,ossClient,byteArrayMap); //
//DownloadUtil.downloadHttp(url, path);
logger.debug("Download:[{}] into: [{}] is done", url, fileName);
latch.countDown();
}
}
/**
* 生成pdf线程
*/
//TODO 生成pdf线程 文件不落地方式
static class CreatedPdfWorkerByte implements Runnable {
private CountDownLatch latch;
private MineDto mineDto;
private Map byteArrayMap;
CreatedPdfWorkerByte(CountDownLatch latch,MineDto mineDto,Map byteArrayMap) {
this.latch = latch;
this.mineDto = mineDto;
this.byteArrayMap = byteArrayMap;
}
@SneakyThrows
@Override
public void run() {
logger.debug("Start batch:[{}] into: [{}]", mineDto.getTbMember().getUsername());
List skillCertList = new ArrayList<>();
skillCertList.add(mineDto.getSkillCert());
byteArrayMap.put(mineDto.getTbMember().getUsername()+mineDto.getTbMember().getId()+System.currentTimeMillis()+".pdf",
PdfUtil.CreatePdfByteArray(mineDto.getTbMember(),mineDto.getEducationalExp(),mineDto.getSocialPractice(),mineDto.getCampusActivities(),skillCertList));
logger.debug("CreatedPdf:[{}] into: [{}] is done", mineDto.getTbMember().getUsername());
latch.countDown();
}
}
}
线程工具类
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class ThreadUtil {
/**
* 创建批量下载线程池
*
* @param threadSize 下载线程数
* @return ExecutorService
*/
public static ExecutorService buildDownloadBatchThreadPool(int threadSize) {
int keepAlive = 0;
String prefix = "download-batch";
ThreadFactory factory = ThreadUtil.buildThreadFactory(prefix);
return new ThreadPoolExecutor(threadSize,
threadSize,
keepAlive,
TimeUnit.SECONDS,
new ArrayBlockingQueue<>(threadSize),
factory);
}
/**
* 创建自定义线程工厂
*
* @param prefix 名称前缀
* @return ThreadFactory
*/
public static ThreadFactory buildThreadFactory(String prefix) {
return new CustomThreadFactory(prefix);
}
/**
* 自定义线程工厂
*/
public static class CustomThreadFactory implements ThreadFactory {
private String threadNamePrefix;
private AtomicInteger counter = new AtomicInteger(1);
/**
* 自定义线程工厂
*
* @param threadNamePrefix 工厂名称前缀
*/
CustomThreadFactory(String threadNamePrefix) {
this.threadNamePrefix = threadNamePrefix;
}
@Override
public Thread newThread(Runnable r) {
String threadName = threadNamePrefix + "-t" + counter.getAndIncrement();
return new Thread(r, threadName);
}
}
}
后端controller层接口代码
@GetMapping("/test/testBatchDownload")
public void testBatchDownload(@RequestParam("id") List id,
@RequestParam("jobId") Integer jobId,
@RequestParam("jobType") Integer jobType,
HttpServletRequest request, HttpServletResponse response) throws IOException {
long startc = System.currentTimeMillis();
List paramVoList = jobApplyService.finJobApplyResume(id, jobId,jobType);
//附件简历
List fileResume = paramVoList.stream().filter(s->s.getResumeId()!=0).collect(Collectors.toList());
//在线简历用户id
List fileOnline = paramVoList.stream().filter(s->s.getResumeId()==0).map(PositionDeliveListVO::getUserId).collect(Collectors.toList());
ByteArrayOutputStream byteArrayOutputStream = null;
ZipOutputStream zip = null;
try {
String resumeFlag = "test"; //附件简历标识
String onlineFlag = ":online"; //在线简历标识
String zipFileName = paramVoList.size()>0 ? paramVoList.get(0).getJobTitle()+"(简历).zip":"简历压缩包.zip";
Map map = fileResume.stream().collect(Collectors.toMap(s->s.getUserId()+resumeFlag+s.getFile(),s->s.getUserName()+s.getUserId()+s.getFile().substring(s.getFile().lastIndexOf("."))));
List mineDtoList = new ArrayList<>();
if(fileOnline.size()>0){
mineDtoList = academicActivitiesService.findByUserIds(fileOnline);
map.putAll(mineDtoList.stream().collect(Collectors.toMap(s->s.getTbMember().getId()+onlineFlag,s->s.getTbMember().getId().toString())));
}
Map mineDtoMap = mineDtoList
.stream().collect(Collectors.toMap(s->s.getTbMember()!=null?s.getTbMember().getId().toString():"-1",s->s));
long starts = System.currentTimeMillis();
OSSClient ossClient = new OSSClient(CloudConstant.ENDPOINT, CloudConstant.ACCESSKEYID, CloudConstant.ACCESSKEYSECRET);
Map mapFileByteArray = DownloadUtil.batchByte(map,mineDtoMap,ossClient,resumeFlag,onlineFlag,new HashMap<>());
ossClient.shutdown();
log.info("(实体文件不落地方式)---远程获取文件数量:{},本地生成文件数:{}",fileResume.size(),fileOnline.size());
log.info("(实体文件不落地方式)---下载耗时》》》》》》》》》》》》》》》》:{}ms",(System.currentTimeMillis() - starts));
byteArrayOutputStream = new ByteArrayOutputStream();
// 用于将数据压缩成Zip文件格式
zip = new ZipOutputStream(byteArrayOutputStream);
long star = System.currentTimeMillis();
for (Map.Entry entry : mapFileByteArray.entrySet()) {
zip.putNextEntry(new ZipEntry(entry.getKey()));
// 向压缩文件中输出数据
zip.write(entry.getValue());
zip.closeEntry(); // 当前文件写完，定位为写入下一条项目
}
log.info("(实体文件不落地方式)--->>>>>>>>>>>>>>>>>>>>>>>>>>文件流装入zip文件耗时:{}ms",(System.currentTimeMillis()-star));
DownloadUtil.createdWriteZip(request,response, zipFileName,byteArrayOutputStream.toByteArray());
log.info("(实体文件不落地方式)--->>>>>>>>>>>>>>>>>>>>>>>>>>总耗时:{}ms",(System.currentTimeMillis()-startc));
}catch(Exception e){
e.printStackTrace();
}finally {
if (null != byteArrayOutputStream){
byteArrayOutputStream.close();
}
if (null != zip){
zip.close();
}
}
}