Java文件切片 & 切片合并代码
研究了下大文件的断点上传，写了个文件切片和切片合并的代码，使用了RandomAccessFile和MappedByteBuffer。
测试了下性能，速度还可以。测试中使用了1.18G的待分割文件，每片最大1M，切片+合并测试了几次，平均耗时为4.8S左右，用的机械硬盘。
代码共享如下，需要的同学拿走。
文件分割器：
package com.lonzh.fileutils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* @author Keven split a src-filt into sevral small pieces. The src-file's size
* if required to be smaller than miPartSize * Integer.MAX_VALUE
*/
public class FileSplitter {
private final String msSrcFile, msDstDir, msOutFileBaseName;
private int miPartSize = 1024 * 1024;
private int miMaxThrPoolSize = 10;
private ThreadPoolExecutor moThrPool;
private FileInputStream moInput;
private FileChannel moInChannel;
/**
* Constructor
*
* @param psSrcFile
* @param psDstDir
* @param psOutFileBaseName the output files will be named by
*/
public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName) {
msSrcFile = psSrcFile;
msDstDir = psDstDir;
msOutFileBaseName = psOutFileBaseName;
}
/**
* Constructor
*
* @param psSrcFile
* @param psDstDir
* @param psOutFileBaseName
* @param piPartSize the split-out file's max-size
*/
public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize) {
msSrcFile = psSrcFile;
msDstDir = psDstDir;
msOutFileBaseName = psOutFileBaseName;
miPartSize = piPartSize;
}
/**
* Constructor
*
* @param psSrcFile
* @param psDstDir
* @param psOutFileBaseName
* @param piPartSize
* @param piMaxThrPool the max-size of thread pool
*/
public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize, int piMaxThrPool) {
msSrcFile = psSrcFile;
msDstDir = psDstDir;
msOutFileBaseName = psOutFileBaseName;
miPartSize = piPartSize;
miMaxThrPoolSize = piMaxThrPool;
}
/**
* Judge if the splitter is working
*
* @return
*/
public boolean isBusy() {
return moThrPool != null && !moThrPool.isTerminated();
}
/**
* Start splitting
*
* @return the array of the names of split-out files
* @throws IOException
*/
public String[] start() throws IOException {
File loFile = new File(msSrcFile);
if (!loFile.exists()) {
throw new IOException("Src-File not found:" + msSrcFile);
}
long liFileLen = loFile.length();
long liPartCnt = liFileLen / miPartSize + (liFileLen % miPartSize == 0 ? 0 : 1);
if (liPartCnt > Integer.MAX_VALUE) {
throw new IOException("Src-File too large");
}
moInput = new FileInputStream(msSrcFile);
moInChannel = moInput.getChannel();
moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue());
String[] laOutFiles = new String[(int) liPartCnt];
for (int i = 0; i < (int) liPartCnt; i++) {
moThrPool.execute(new SplitThread(i, i == liPartCnt - 1 ? liFileLen % miPartSize : miPartSize));
laOutFiles[i] = msDstDir + File.separator + msOutFileBaseName + "." + i + ".part";
}
moThrPool.shutdown();
// Monitor
new Thread() {
@Override
public void run() {
while (true) {
if (moThrPool.isTerminated()) {
if (moInChannel != null) {
try {
moInChannel.close();
} catch (IOException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (moInput != null) {
try {
moInput.close();
} catch (IOException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
break;
}
try {
Thread.sleep(100);
} catch (InterruptedException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}.start();
return laOutFiles;
}
/**
* Splitting Thread
*/
private class SplitThread implements Runnable {
private final int miPartIndex;
private final long miSize;
public SplitThread(int piPartIndex, long piSize) {
miPartIndex = piPartIndex;
miSize = piSize;
}
@Override
public void run() {
String lsDstPartFile = msDstDir + File.separator + msOutFileBaseName + "." + miPartIndex + ".part";
System.out.println("FileSplitter: splitting " + lsDstPartFile);
long liStartTime = System.currentTimeMillis();
// output vars
File loDstFile = new File(lsDstPartFile);
RandomAccessFile loOutput = null;
FileChannel loOutChannel = null;
try {
// Input Map
MappedByteBuffer loInBuf = moInChannel.map(FileChannel.MapMode.READ_ONLY, miPartSize * miPartIndex, miSize);
// Output Map
if (!loDstFile.exists()) {
loDstFile.createNewFile();
}
loOutput = new RandomAccessFile(loDstFile, "rw");
loOutChannel = loOutput.getChannel();
MappedByteBuffer loOutBuf = loOutChannel.map(FileChannel.MapMode.READ_WRITE, 0, miSize);
byte liByte;
while (loInBuf.hasRemaining()) {
liByte = loInBuf.get();
loOutBuf.put(liByte);
}
} catch (IOException ex) {
System.out.print("part_index:" + miPartIndex + " boom");
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
} finally {
// close output
if (loOutChannel != null) {
try {
loOutChannel.close();
} catch (IOException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (loOutput != null) {
try {
loOutput.close();
} catch (IOException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
System.out.println("Splitting Task " + lsDstPartFile + " Terminated. Spend " + (System.currentTimeMillis() - liStartTime) + " milsecs");
}
}
}
package com.lonzh.fileutils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* @author Keven
*/
public class FileUnifier {
private final String[] maSrcFiles;
private final String msDstFile;
private int miMaxThrPoolSize = 10;
private ThreadPoolExecutor moThrPool;
private RandomAccessFile moOutput;
private FileChannel moOutputChannel;
/**
* Constructor
*
* @param paSrcFiles
* @param psDstFile
*/
public FileUnifier(String[] paSrcFiles, String psDstFile) {
maSrcFiles = paSrcFiles;
msDstFile = psDstFile;
}
/**
* Constructor
*
* @param paSrcFiles
* @param psDstFile
* @param piMaxThrPool
*/
public FileUnifier(String[] paSrcFiles, String psDstFile, int piMaxThrPool) {
maSrcFiles = paSrcFiles;
msDstFile = psDstFile;
miMaxThrPoolSize = piMaxThrPool;
}
/**
* Judge if the unifier is working
*
* @return
*/
public boolean isBusy() {
return moThrPool != null && !moThrPool.isTerminated();
}
/**
* Start unifying
*
* @throws java.io.IOException
*/
public void start() throws IOException {
File loOutFile = new File(msDstFile);
if (!loOutFile.exists()) {
loOutFile.createNewFile();
}
moOutput = new RandomAccessFile(loOutFile, "rw");
moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue());
// get total size
long[][] laOffsetSizes = new long[maSrcFiles.length][2];
long liOffset = 0;
for (int i = 0; i < maSrcFiles.length; i++) {
String lsSrcFile = maSrcFiles[i];
File loFile = new File(lsSrcFile);
if (!loFile.exists()) {
throw new IOException("Src-File not exist:" + lsSrcFile);
}
laOffsetSizes[i][0] = liOffset;
laOffsetSizes[i][1] = loFile.length();
liOffset += loFile.length();
}
moOutput.setLength(liOffset);
moOutputChannel = moOutput.getChannel();
for (int i = 0; i < maSrcFiles.length; i++) {
moThrPool.execute(new UnifyThread(maSrcFiles[i], laOffsetSizes[i][0], laOffsetSizes[i][1]));
}
moThrPool.shutdown();
// Start monitor Thread
new Thread() {
@Override
public void run() {
while (true) {
if (moThrPool.isTerminated()) {
if (moOutputChannel != null) {
try {
moOutputChannel.close();
} catch (IOException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (moOutput != null) {
try {
moOutput.close();
} catch (IOException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
}
}
break;
}
try {
Thread.sleep(100);
} catch (InterruptedException ex) {
Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}.start();
}
/**
* Unifying thread
*/
private class UnifyThread implements Runnable {
private final String msSrcFile;
private final long miOffset, miSize;
public UnifyThread(String psSrcFile, long piOffset, long piSize) {
msSrcFile = psSrcFile;
miOffset = piOffset;
miSize = piSize;
}
@Override
public void run() {
File loSrcFile = new File(msSrcFile);
FileInputStream loInput = null;
FileChannel loInChannel = null;
try {
System.out.println("FileUnifier: start unifying " + msSrcFile);
long liStartTime = System.currentTimeMillis();
loInput = new FileInputStream(loSrcFile);
loInChannel = loInput.getChannel();
MappedByteBuffer loInBuf = loInChannel.map(FileChannel.MapMode.READ_ONLY, 0, miSize);
MappedByteBuffer loOutBuf = moOutputChannel.map(FileChannel.MapMode.READ_WRITE, miOffset, miSize);
while (loInBuf.hasRemaining()) {
loOutBuf.put(loInBuf.get());
}
System.out.println("FileUnifier: unify " + msSrcFile + " complete. Spend " + (System.currentTimeMillis() - liStartTime) + "milsecs");
} catch (FileNotFoundException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
} finally {
// Close input
if (loInChannel != null) {
try {
loInChannel.close();
} catch (IOException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
}
}
if (loInput != null) {
try {
loInput.close();
} catch (IOException ex) {
Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
}
}
}
测试代码：
package test_java;
import com.lonzh.fileutils.FileSplitter;
import com.lonzh.fileutils.FileUnifier;
import java.io.File;
import java.io.IOException;
/**
*
* @author Keven
*/
public class Test_java {
/**
* @param args the command line arguments
* @throws java.io.IOException
* @throws java.lang.InterruptedException
*/
public static void main(String[] args) throws IOException, InterruptedException {
String lsSrcFile = "D:\\奔腾年代.rmvb";
String lsDstDir = "D:\\parts";
String lsDstFile = "D:\\parts\\dst.rmvb";
File loDstFile = new File(lsDstDir);
if (!loDstFile.exists()) {
loDstFile.mkdir();
}
long liStartTime = System.currentTimeMillis();
FileSplitter loSplitter = new FileSplitter(lsSrcFile, lsDstDir, "test_cut");
String[] laPartFiles = loSplitter.start();
while (loSplitter.isBusy()) {
Thread.sleep(200);
}
FileUnifier loUnifier = new FileUnifier(laPartFiles, lsDstFile);
loUnifier.start();
while (loUnifier.isBusy()) {
Thread.sleep(200);
}
System.out.println("done, spend " + (System.currentTimeMillis() - liStartTime) + " milsecs");
}
}