大文件分片上传、断点续传（附带前后端demo）
序
大文件上传，断点续传，秒传，作为高频考察的技术点，大多数人都是知其然而不知其所以然，下面我们从前后端一起的角度来探究一番，
相信只要你肯花一点时间认真地理解它，你就会发现你花了一些时间。。。
思路图
前端核心代码片段
按照指定size进行文件切片
createChunk(file, size = 512 * 1024) {
const chunkList = [];
let cur = 0;
while (cur < file.size) {
// 使用slice方法切片
chunkList.push({ file: file.slice(cur, cur + size) });
cur += size;
}
return chunkList;
}
文件使用js-md5转哈希
handleFileChange(e) {
let fileReader = new FileReader();
fileReader.readAsDataURL(e.target.files[0]);
fileReader.onload = function (e2) {
const hexHash = md5(e2.target.result)+'.'+that.fileObj.file.name.split('.').pop();
};
}
切片转成表单对象后，配置成一个请求列表。
const requestList = noUploadChunks.map(({ file, fileName, index, chunkName }) => {
const formData = new FormData();
formData.append("file", file);
formData.append("fileName", fileName);
formData.append("chunkName", chunkName);
return { formData, index };
})
.map(({ formData, index }) =>
axiosRequest({
url: "http://localhost:3000/upload",
data: formData
})
)
前端完整代码
<template>
<div>
<input type="file" @change="handleFileChange" />
<el-button @click="uploadChunks"> 上传 </el-button>
<!-- <el-button @click="pauseUpload"> 暂停 </el-button> -->
<div style="width: 300px">
总进度：
<el-progress :percentage="tempPercent"></el-progress>
切片进度：
<div v-for="(item,index) in fileObj.chunkList" :key="index">
<span>{{ item.chunkName }}：</span>
<el-progress :percentage="item.percent"></el-progress>
</div>
</div>
</div>
</template>
<script>
import axios from "axios";
import md5 from 'js-md5'
const CancelToken = axios.CancelToken;
let source = CancelToken.source();
function axiosRequest({
url,
method = "post",
data,
headers = {},
onUploadProgress = (e) => e, // 进度回调
}) {
return new Promise((resolve, reject) => {
axios[method](url, data, {
headers,
onUploadProgress, // 传入监听进度回调
cancelToken: source.token
})
.then((res) => {
resolve(res);
})
.catch((err) => {
reject(err);
});
});
}
export default {
data() {
return {
fileObj: {
file: null,
chunkList:[]
},
tempPercent:0
};
},
methods: {
handleFileChange(e) {
const [file] = e.target.files;
if (!file) return;
this.fileObj.file = file;
const fileObj = this.fileObj;
if (!fileObj.file) return;
const chunkList = this.createChunk(fileObj.file);
console.log(chunkList); // 看看chunkList长什么样子
let that = this
// 获取此视频的哈希值,作为名字
let fileReader = new FileReader();
fileReader.readAsDataURL(e.target.files[0]);
fileReader.onload = function (e2) {
const hexHash = md5(e2.target.result)+'.'+that.fileObj.file.name.split('.').pop();
that.fileObj.name = hexHash
that.fileObj.chunkList = chunkList.map(({ file }, index) => ({
file,
size: file.size,
percent: 0,
chunkName: `${hexHash}-${index}`,
fileName: hexHash,
index,
}))
};
},
createChunk(file, size = 512 * 1024) {
const chunkList = [];
let cur = 0;
while (cur < file.size) {
// 使用slice方法切片
chunkList.push({ file: file.slice(cur, cur + size) });
cur += size;
}
return chunkList;
},
async uploadChunks() {
const {uploadedList,shouldUpload} = await this.verifyUpload()
// 如果存在这个文件
if(!shouldUpload){
console.log('秒传成功')
this.fileObj.chunkList.forEach(item => {
item.percent = 100
});
return
}
let noUploadChunks = []
if(uploadedList&&uploadedList.length>0 &&uploadedList.length !== this.fileObj.chunkList.length){
// 如果存在切片。只上传没有的切片
noUploadChunks = this.fileObj.chunkList.filter(item=>{
if(uploadedList.includes(item.chunkName)){
item.percent = 100
}
return !uploadedList.includes(item.chunkName)
})
}else{
noUploadChunks = this.fileObj.chunkList
}
const requestList = noUploadChunks.map(({ file, fileName, index, chunkName }) => {
const formData = new FormData();
formData.append("file", file);
formData.append("fileName", fileName);
formData.append("chunkName", chunkName);
return { formData, index };
})
.map(({ formData, index }) =>
axiosRequest({
url: "http://localhost:3000/upload",
data: formData,
onUploadProgress: this.createProgressHandler(
this.fileObj.chunkList[index]
), // 传入监听上传进度回调
})
);
await Promise.all(requestList); // 使用Promise.all进行请求
this.mergeChunks()
},
createProgressHandler(item) {
return (e) => {
// 设置每一个切片的进度百分比
item.percent = parseInt(String((e.loaded / e.total) * 100));
};
},
mergeChunks(size = 512 * 1024) {
axiosRequest({
url: "http://localhost:3000/merge",
headers: {
"content-type": "application/json",
},
data: JSON.stringify({
size,
fileName: this.fileObj.name
}),
});
},
pauseUpload() {
source.cancel("中断上传!");
source = CancelToken.source(); // 重置source，确保能续传
},
async verifyUpload () {
const { data } = await axiosRequest({
url: "http://localhost:3000/verify",
headers: {
"content-type": "application/json",
},
data: JSON.stringify({
fileName:this.fileObj.name
}),
});
return data
}
},
computed: {
totalPercent() {
const fileObj = this.fileObj;
if (fileObj.chunkList.length === 0) return 0;
const loaded = fileObj.chunkList
.map(({ size, percent }) => size * percent)
.reduce((pre, next) => pre + next);
return parseInt((loaded / fileObj.file.size).toFixed(2));
},
},
watch: {
totalPercent (newVal) {
if (newVal > this.tempPercent) this.tempPercent = newVal
}
},
};
</script>
<style lang="scss" scoped></style>
服务端接收代码（用node.js模拟）
const http = require("http");
const path = require("path");
const fse = require("fs-extra");
const multiparty = require("multiparty");
const server = http.createServer();
const UPLOAD_DIR = path.resolve(__dirname, ".", `qiepian`); // 切片存储目录
server.on("request", async (req, res) => {
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Headers", "*");
res.setHeader('Content-Type','text/html; charset=utf-8');
if (req.method === "OPTIONS") {
res.status = 200;
res.end();
return;
}
console.log(req.url);
if (req.url === "/upload") {
const multipart = new multiparty.Form();
multipart.parse(req, async (err, fields, files) => {
if (err) {
console.log("errrrr", err);
return;
}
const [file] = files.file;
const [fileName] = fields.fileName;
const [chunkName] = fields.chunkName;
// 保存切片的文件夹的路径，比如 张远-嘉宾.flac-chunks
const chunkDir = path.resolve(UPLOAD_DIR, `${fileName}-chunks`);
// // 切片目录不存在，创建切片目录
if (!fse.existsSync(chunkDir)) {
await fse.mkdirs(chunkDir);
}
// 把切片移动到切片文件夹
await fse.move(file.path, `${chunkDir}/${chunkName}`);
res.end(
JSON.stringify({
code: 0,
message: "切片上传成功",
})
);
});
}
// 合并切片
// 接收请求的参数
const resolvePost = (req) =>
new Promise((res) => {
let chunk = "";
req.on("data", (data) => {
chunk += data;
});
req.on("end", () => {
res(JSON.parse(chunk));
});
});
const pipeStream = (path, writeStream) => {
console.log("path", path);
return new Promise((resolve) => {
const readStream = fse.createReadStream(path);
readStream.on("end", () => {
// fse.unlinkSync(path); // 删除切片文件
resolve();
});
readStream.pipe(writeStream);
});
};
// 合并切片
const mergeFileChunk = async (filePath, fileName, size) => {
// filePath：你将切片合并到哪里，的路径
const chunkDir = path.resolve(UPLOAD_DIR, `${fileName}-chunks`);
let chunkPaths = null;
// 获取切片文件夹里所有切片，返回一个数组
chunkPaths = await fse.readdir(chunkDir);
// 根据切片下标进行排序
// 否则直接读取目录的获得的顺序可能会错乱
chunkPaths.sort((a, b) => a.split("-")[1] - b.split("-")[1]);
const arr = chunkPaths.map((chunkPath, index) => {
return pipeStream(
path.resolve(chunkDir, chunkPath),
// 指定位置创建可写流
fse.createWriteStream(filePath, {
start: index * size,
end: (index + 1) * size,
})
);
});
await Promise.all(arr);
};
if (req.url === "/merge") {
const data = await resolvePost(req);
const { fileName, size } = data;
const filePath = path.resolve(UPLOAD_DIR, fileName);
await mergeFileChunk(filePath, fileName, size);
res.end(
JSON.stringify({
code: 0,
message: "文件合并成功",
})
);
}
if (req.url === "/verify") {
// 返回已经上传切片名列表
const createUploadedList = async fileName =>
fse.existsSync(path.resolve(UPLOAD_DIR, fileName))
? await fse.readdir(path.resolve(UPLOAD_DIR, fileName))
: [];
const data = await resolvePost(req);
const { fileName } = data;
const filePath = path.resolve(UPLOAD_DIR, fileName);
console.log(filePath)
if (fse.existsSync(filePath)) {
res.end(
JSON.stringify({
shouldUpload: false
})
);
} else {
res.end(
JSON.stringify({
shouldUpload: true,
uploadedList: await createUploadedList(`${fileName}-chunks`)
})
);
}
}
});
server.listen(3000, () => console.log("正在监听 3000 端口"));
demo测试
- npm安装好缺的依赖，
- 运行前端代码。
- node xxx.js 运行后端代码。