测试代码
package com.boot.demo.test.io;
import java.io.*;
import java.lang.reflect.Method;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.security.AccessController;
import java.security.PrivilegedAction;
/**
* @author braska
* @date 2020/3/19
**/
public class FileTest {
public static void fileStream(String sourceFile, String targetFile) {
File file = new File(targetFile);
try (FileInputStream fis = new FileInputStream(sourceFile);
FileOutputStream fos = new FileOutputStream(file)) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = fis.read(bytes)) > 0) {
fos.write(bytes, 0, len);
}
} catch (Exception e) {
}
}
public static void bufferStream(String sourceFile, String targetFile) {
try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(Paths.get(sourceFile)));
BufferedOutputStream bos =
new BufferedOutputStream(Files.newOutputStream(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE))) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = bis.read(bytes)) > 0) {
bos.write(bytes, 0, len);
}
} catch (Exception e) {
e.printStackTrace();
}
}
public static void randomFile(String sourceFile, String targetFile) {
try (RandomAccessFile read = new RandomAccessFile(sourceFile, "r");
RandomAccessFile write = new RandomAccessFile(targetFile, "rw")) {
byte[] bytes = new byte[1024 * 1024];
int len;
while ((len = read.read(bytes)) > 0) {
write.write(bytes, 0, len);
}
} catch (Exception e) {
}
}
public static void memoryMap(String sourceFile, String targetFile) {
try (FileChannel rc = FileChannel.open(Paths.get(sourceFile));
FileChannel wc = FileChannel.open(Paths.get(targetFile),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
long copy = 1L << 30;
long cur = 0;
long fileLength = rc.size();
while (cur < fileLength) {
copy = cur + copy > fileLength ? (fileLength - cur) : copy;
MappedByteBuffer rMap = rc.map(FileChannel.MapMode.READ_ONLY, cur, copy);
MappedByteBuffer wMap = wc.map(FileChannel.MapMode.READ_WRITE, cur, copy);
for (int i = 0; i < copy; i++) {
byte b = rMap.get(i); //从源文件读取字节
wMap.put(i, b); //把字节写到目标文件中
}
System.gc(); //手动调用 GC <必须的，否则出现异常>
System.runFinalization();
cur += copy;
}
} catch (Exception e) {
e.printStackTrace();
}
}
private static String buildFilePath(String path, String fileName, String extension) {
return String.format("%s%s.%s", path, fileName, extension);
}
public static void main(String[] args) {
/* String path = "F:\\workspace\\demo\\";
String extension = "hprof";
// 30M文件
String sourceFile = buildFilePath(path, "01", extension);*/
/* String path = "E:\\software\\";
String extension = "exe";
// 460M文件
String sourceFile = buildFilePath(path, "Anaconda3-2019.10-Windows-x86_64", extension);*/
String path = "E:\\software\\";
String extension = "zip";
// 1.47G文件
String sourceFile = buildFilePath(path, "software", extension);
String targetFile;
long start;
/* targetFile = buildFilePath(path, "target_file_stream", extension);
start = System.currentTimeMillis();
FileTest.fileStream(sourceFile, targetFile);
System.out.println("file stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_buffer_stream", extension);
start = System.currentTimeMillis();
FileTest.bufferStream(sourceFile, targetFile);
System.out.println("buffer stream used time:" + (System.currentTimeMillis() - start));*/
/* targetFile = buildFilePath(path, "target_random_file", extension);
start = System.currentTimeMillis();
FileTest.randomFile(sourceFile, targetFile);
System.out.println("random file used time:" + (System.currentTimeMillis() - start));*/
targetFile = buildFilePath(path, "target_memory_map", extension);
start = System.currentTimeMillis();
FileTest.memoryMap(sourceFile, targetFile);
System.out.println("memory map used time:" + (System.currentTimeMillis() - start));
}
}
测试结果
| 文件大小 | 读写方式 | 耗时 |
|---|---|---|
| 30M | 普通文件流 | 50-60 ms |
| 缓存流 | 32-35 ms | |
| 随机文件方式 | 40-50 ms | |
| 内存映射文件 | 50-60 ms | |
| 461M | 普通文件流 | 1300-2300 ms |
| 缓存流 | 1700-2000 ms | |
| 随机文件方式 | 1300-3000 ms | |
| 内存映射文件 | 890-1000 ms | |
| 1.47G | 普通文件流 | 11s |
| 缓存流 | 9s | |
| 随机文件方式 | 10s | |
| 内存映射文件 | 3s(首次较慢) |
结尾：测试1.47G大文件时，内存映射文件中copy大小做了调整，当copy为1G时(copy=1L<<30)性能最佳，整过过程1-3秒左右。调至128M、512M。大约耗时都在15秒左右。为了公平起见，把其他方法中的byte缓冲区大小也做了同样调整，耗时变化不大。有些甚至更慢。
封装MappedBuffer工具类，代码如下：
package com.boot.demo.test.io;
import java.io.Closeable;
import java.io.IOException;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
* @author braska
* @date 2020/3/20
**/
public class MappedByteBufferReader implements Closeable {
private static final int BUFFERED_SIZE = 1 << 27;
private static final FileChannel.MapMode MAP_MODE = FileChannel.MapMode.READ_ONLY;
private FileChannel fileChannel;
private final long fileSize;
private final int bufferedSize;
BlockingQueue queue;
public MappedByteBufferReader(String file) throws Exception {
this(file, BUFFERED_SIZE, MAP_MODE);
}
public MappedByteBufferReader(String file, int bufferedSize) throws Exception {
this(file, bufferedSize, MAP_MODE);
}
public MappedByteBufferReader(String file, FileChannel.MapMode mapMode) throws Exception {
this(file, BUFFERED_SIZE, mapMode);
}
public MappedByteBufferReader(String file, int bufferedSize, FileChannel.MapMode mapMode) throws Exception {
this.fileChannel = FileChannel.open(Paths.get(file));
this.fileSize = fileChannel.size();
this.bufferedSize = bufferedSize;
int capacity = (int) Math.ceil((double) fileSize / (double) bufferedSize);
this.queue = new ArrayBlockingQueue(capacity);
long readSize = bufferedSize;
long cursor = 0l;
while (cursor < fileSize) {
readSize = cursor + readSize > fileSize ? fileSize - cursor : readSize;
queue.add(
fileChannel.map(mapMode, cursor, readSize)
);
cursor += readSize;
}
}
public byte[] read() {
byte[] bytes;
MappedByteBuffer byteBuffer = queue.poll();
if (byteBuffer != null) {
int limit = byteBuffer.limit();
int position = byteBuffer.position();
int realSize = this.bufferedSize;
if (limit - position < this.bufferedSize) {
realSize = limit - position;
}
bytes = new byte[realSize];
byteBuffer.get(bytes);
byteBuffer.clear();
return bytes;
}
return null;
}
public long size() {
return this.fileSize;
}
@Override
public void close() throws IOException {
if (this.fileChannel != null) {
this.fileChannel.close();
}
}
public static void main(String[] args) {
long start = System.currentTimeMillis();
try (MappedByteBufferReader reader = new MappedByteBufferReader("E:\\software\\software.zip", 1 << 30);
FileChannel wc = FileChannel.open(Paths.get("E:\\software\\software_reader.zip"),
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE)) {
byte[] data;
MappedByteBuffer writer = wc.map(FileChannel.MapMode.READ_WRITE, 0, reader.size());
while ((data = reader.read()) != null) {
writer.put(data);
}
writer.clear();
System.out.println("used times: " + (System.currentTimeMillis() - start));
} catch (Exception e) {
e.printStackTrace();
}
}
}