vue + python实现大文件分片上传功能
前端主要代码：
主要查看上传相关。。
vue文件：
<template>
<vxe-modal
v-model="show"
class="add-mark-dialog"
:title="title"
width="650"
height="600"
:show-footer="true"
destroy-on-close
@close="handleClose"
>
<div class="main-modal-body">
<div class="form-wrap">
<el-form ref="form" :model="form" :rules="rules" label-width="140px">
<el-row>
<el-col :span="24">
<el-form-item label="任务名称：" prop="task_name">
<el-input v-model="form.task_name" placeholder="请输入任务名称"></el-input>
</el-form-item>
</el-col>
<el-col :span="24">
<el-form-item v-if="type == 'add'" label="上传文件：" prop="file_address">
<el-upload action :auto-upload="false" :show-file-list="false" :on-change="handleChange">
<div class="el-upload__text" style="color:#409EFF;"><i class="el-icon-upload"></i><em>点击上传</em></div>
<div class="el-upload__tip" slot="tip">只能上传zip格式文件，且大小不超过 10 GB 的视频</div>
</el-upload>
<div class="progress-box">
<!-- <span>上传进度：{{ percent.toFixed() }}%</span> -->
<el-progress :percentage="percent"></el-progress>
<!-- <el-button type="primary" size="mini" @click="handleClickBtn">{{ upload | btnTextFilter }}</el-button> -->
</div>
</el-form-item>
<el-form-item v-else-if="type == 'edit'" label="已上传文件：" prop="file_address">
<span style="word-break: break-all" v-for="(file, index) in imgUrlList" :key="index">{{
file.real_address
}}</span>
</el-form-item>
</el-col>
</el-row>
</el-form>
</div>
</div>
<template v-slot:footer>
<div class="a-r">
<el-button @click="handleClose">取消</el-button>
<el-button type="primary" @click="handleSubmit" :loading="btnLoading" :disabled="btnLoading">保存</el-button>
</div>
</template>
</vxe-modal>
</template>
<script>
import SparkMD5 from 'spark-md5'
import axios from 'axios'
import { mapState } from 'vuex'
import util from '@/libs/util'
import { upload, createImgMarkTask, updateImgMarkTask, chunkMerge } from '../api'
const defaultForm = {
task_name: ''
}
export default {
name: 'AddModal',
components: {},
props: {
},
filters: {
btnTextFilter(val) {
return val ? '暂停' : '继续'
}
},
data() {
return {
title: '新增',
type: 'add', // add edit view
btnLoading: false,
show: false,
form: Object.assign({}, defaultForm),
detail: null,
tagarr: [],
addTagVisible: false,
typearr: [],
addTypeVisible: false,
inputValue: '',
inputValue1: '',
upHeaders: null,
fileList: [], // el-upload绑定值
imgUrlList: [], // 真实上传值
fileAddress: '', // 手动输入上传文件地址
rules: {
task_name: [
{ required: true, message: '请输入任务名称', trigger: 'blur' },
{ min: 2, max: 20, message: '长度在 2 到 20 个字符', trigger: 'blur' }
]
},
percent: 0,
// fileAddress: '',
upload: true,
percentCount: 0
}
},
computed: {
...mapState('admin', {
base_url: state => state.settings.base_url
})
},
watch: {},
methods: {
async handleChange(file) {
if (!file) return
this.percent = 0
this.fileAddress = ''
// 获取文件并转成 ArrayBuffer 对象
const fileObj = file.raw
let buffer
try {
buffer = await this.fileToBuffer(fileObj)
} catch (e) {
console.log(e)
}
// 将文件按固定大小（2M）进行切片，注意此处同时声明了多个常量
const chunkSize = 2097152
const chunkList = [] // 保存所有切片的数组
const chunkListLength = Math.ceil(fileObj.size / chunkSize) // 计算总共多个切片
const suffix = /\.([0-9A-z]+)$/.exec(fileObj.name)[1] // 文件后缀名
// 根据文件内容生成 hash 值
const spark = new SparkMD5.ArrayBuffer()
spark.append(buffer)
const hash = spark.end()
// 生成切片，这里后端要求传递的参数为字节数据块（chunk）和每个数据块的文件名（fileName）
let curChunk = 0 // 切片时的初始位置
for (let i = 0; i < chunkListLength; i++) {
const item = {
chunk: fileObj.slice(curChunk, curChunk + chunkSize),
fileName: `${hash}_${i}.${suffix}`, // 文件名规则按照 hash_1.jpg 命名
chunkNumber: i, // 当前分片索引
totalChunks: chunkListLength, // 总共分片
identifier: hash // 文件hash 唯一值
}
curChunk += chunkSize
chunkList.push(item)
}
this.chunkList = chunkList // sendRequest 要用到
this.hash = hash // sendRequest 要用到
this.sendRequest()
},
// 发送请求
sendRequest() {
const requestList = [] // 请求集合
this.chunkList.forEach((item, index) => {
const fn = () => {
const formData = new FormData()
formData.append('chunk', item.chunk)
formData.append('fileName', item.fileName)
formData.append('chunkNumber', item.chunkNumber)
formData.append('totalChunks', item.totalChunks)
formData.append('identifier', item.identifier)
return axios({
baseURL: util.baseURL(),
url: '/api/img_mark_task/chunk_upload/',
method: 'post',
headers: { 'Content-Type': 'multipart/form-data', Authorization: 'JWT ' + util.cookies.get('token') },
data: formData
}).then(res => {
console.log('res=====1', res)
if (res.data.code === 2000) {
// 成功
if (this.percentCount === 0) {
// 避免上传成功后会删除切片改变 chunkList 的长度影响到 percentCount 的值
this.percentCount = 100 / this.chunkList.length
}
this.percent += this.percentCount // 改变进度
this.chunkList.splice(index, 1) // 一旦上传成功就删除这一个 chunk，方便断点续传
}
})
}
requestList.push(fn)
})
let i = 0 // 记录发送的请求个数
// 文件切片全部发送完毕后，需要请求 '/merge' 接口，把文件的 hash 传递给服务器
const complete = () => {
chunkMerge({ hash: this.hash, totalChunks: requestList.length }).then(res => {
console.log('res===', res)
this.fileAddress = res.data.url
})
// axios({
// url: '/merge',
// method: 'get',
// params: { hash: this.hash }
// }).then(res => {
// if (res.data.code === 0) {
// // 请求发送成功
// this.fileAddress = res.data.path
// }
// })
}
const send = async () => {
if (!this.upload) return
if (i >= requestList.length) {
// 发送完毕
complete()
return
}
await requestList[i]()
i++
send()
}
send() // 发送请求
},
// 按下暂停按钮
handleClickBtn() {
this.upload = !this.upload
// 如果不暂停则继续上传
if (this.upload) this.sendRequest()
},
// 将 File 对象转为 ArrayBuffer
fileToBuffer(file) {
return new Promise((resolve, reject) => {
const fr = new FileReader()
fr.onload = e => {
resolve(e.target.result)
}
fr.readAsArrayBuffer(file)
fr.onerror = () => {
reject(new Error('转换文件格式发生错误'))
}
})
},
// 执行人保存
handleSubmit() {
},
}
}
</script>
<style scoped lang="scss">
.main-modal-body {
.form-wrap {
padding: 16px 0 0 0;
}
}
</style>
<style>
.add-mark-dialog.vxe-modal--wrapper.type--modal .vxe-modal--body {
padding: 0;
}
</style>
以上方案，在上传超过几个G的大文件时，会报错，原因是计算文件的MD5，浏览器内存支撑不住。
// 根据文件内容生成 hash 值,方式2 改进了大文件的计算，分片计算，否则内存溢出
let hash = ''
try {
const tempMD5Info = await this.fileMD5(fileObj)
hash = tempMD5Info.fileMd5
// console.log('new_md5=', tempMD5Info.fileMd5)
} catch (e) {
console.log(e)
}
// 解决大文件分片计算MD5值问题
fileMD5(files) {
// const pieceSize = 2097152 // 2MB
const pieceSize = 1048576 * 500 // 500MB
// const spark = new SparkMD5()
const spark = new SparkMD5.ArrayBuffer()
const loading = this.$loading({
lock: true,
text: '努力处理中...',
spinner: 'el-icon-loading',
background: 'rgba(0, 0, 0, 0.7)'
})
return new Promise((resolve, reject) => {
const fileReader = new FileReader()
const piece = Math.ceil(files.size / pieceSize)
const nextPiece = () => {
const start = currentPieces * pieceSize
const end = (start + pieceSize) >= files.size ? files.size : start + pieceSize
fileReader.readAsArrayBuffer(files.slice(start, end))
}
let currentPieces = 0
fileReader.onload = event => {
const e = window.event || event
spark.append(e.target.result)
currentPieces++
if (currentPieces < piece) {
nextPiece()
} else {
resolve({ fileName: files.name, fileMd5: spark.end() })
loading.close()
}
}
fileReader.onerror = err => {
reject(err)
console.log(err)
loading.close()
}
nextPiece()
})
},
将计算md5方法，修改为分片读取，最后算出文件md5值。
另外，原来的分片上传，是一个接口请求完后再发一个接口请求，比较慢。
// 优化，一次发送一个请求改为一次发送2个请求
const send = async () => {
if (!this.upload) return
if (i >= requestList.length) {
// 发送完毕
complete()
return
}
if ((i + 1) >= requestList.length) {
await requestList[i]()
} else {
await Promise.all([requestList[i](), requestList[i + 1]()])
}
i = i + 2
send()
发送部分代码，修改为一次发送2个请求，这样可以提高速率。 当然，你也可以改为一次发送3个。都行。
后端接口：
upload 上传 和 合并 两个接口
# 大文件分片上传
def deldir(dir):
if not os.path.exists(dir):
return False
if os.path.isfile(dir):
os.remove(dir)
return
for i in os.listdir(dir):
t = os.path.join(dir, i)
if os.path.isdir(t):
deldir(t)#重新调用次方法
else:
os.unlink(t)
os.removedirs(dir)#递归删除目录下面的空文件夹
class ChunkUploadViewSet(CustomModelViewSet):
def chunk_upload(self, request, *args, **kwargs):
file_name = request.POST.get('identifier')
chunk_index = int(request.POST.get("chunkNumber"))
total_chunk = int(request.POST.get("totalChunks"))
upload_file = request.FILES["chunk"] # 二进制数据
file_path = os.path.join(settings.MEDIA_ROOT, 'chunk_file', file_name)
chunk_path = os.path.join(file_path, str(chunk_index))
if not os.path.exists(file_path):
os.makedirs(file_path)
with open(chunk_path, 'wb+') as destination:
for chunk in upload_file.chunks():
destination.write(chunk)
# print('file_name====',upload_file)
res = {
'file_name': file_name,
"chunk_index": chunk_index,
'status': 1
}
return DetailResponse(data=res, msg="获取成功")
# 合并分片
def chunk_merge(self, request, *args, **kwargs):
file_name = request.GET.get("hash") # 文件hash
total_chunk = int(request.GET.get("totalChunks")) # 总共分片
file_path = os.path.join(settings.MEDIA_ROOT, 'chunk_file', file_name)
chunks_list = list(set(os.listdir(file_path)))
is_over = False
# print('total_chunk===', total_chunk, 'chunks_list', len(chunks_list))
if len(chunks_list) == total_chunk:
is_over = True
if is_over:
# 所有的分片 必须按照分块顺序排序，否则 可能合并的文件顺序被打乱
all_chunk = os.listdir(file_path)
all_chunk.sort(key=lambda x: int(x)) # fig bug: 默认是按 '0' '11'这种字符串类型排序，会导致分片顺序错乱。
target_path = os.path.join(settings.MEDIA_ROOT, 'chunk_file', file_name+".zip")
target_path_temp = os.path.join(settings.MEDIA_ROOT, 'chunk_file', file_name+"temp")
with open(target_path, "wb+") as f:
for chunk in all_chunk:
chunk_path = os.path.join(file_path, chunk)
with open(chunk_path, "rb") as g:
data = g.read()
f.write(data)
deldir(file_path)
# print('file_name====', file_name)
file_url = os.sep.join([settings.MEDIA_ROOT, 'chunk_file', file_name+".zip"])
res = {
"url": file_url, # os.path.join(settings.MEDIA_ROOT, 'chunk_file', file_name+".zip")
"fileName": file_name,
}
return DetailResponse(data=res, msg="获取成功")
我这里写死了上传的文件为zip文件。
参考文章地址：https://www.jianshu.com/p/08524828f84b
贴一个node + vue实现分片上传的文章，参考promise控制部分。
【nodeJs + js 大文件分片上传】
https://my.oschina.net/u/4347428/blog/4468437