模拟kafka测试零拷贝和磁盘顺序写的速度
接收端
使用nio + io多路复用 + 零拷贝 + 磁盘顺序写
package com.chun.multiplexing;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.*;
import java.util.Iterator;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
/**
* @author chun
* @date 2023/4/7 10:14
*/
public class FileReceiver {
private static final int BUFFER_SIZE = 1024 * 1024 * 64;
public static void main(String[] args) {
ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(20);
try {
Selector selector = Selector.open();
ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
serverSocketChannel.bind(new InetSocketAddress(9999));
serverSocketChannel.configureBlocking(false);
serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);
while (true) {
int nkeys = selector.select();
if (nkeys > 0) {
Iterator<SelectionKey> selectionKeys = selector.selectedKeys().iterator();
while (selectionKeys.hasNext()) {
SelectionKey key = selectionKeys.next();
selectionKeys.remove();
if (key.isAcceptable()) {
SocketChannel socketChannel = serverSocketChannel.accept();
socketChannel.configureBlocking(false);
socketChannel.register(selector, SelectionKey.OP_READ);
}
if (key.isValid() && key.isReadable()) {
//当有读取事件发生时，从 SocketChannel 中读取文件信息，并创建独立的线程进行处理
SocketChannel socketChannel = (SocketChannel) key.channel();
key.cancel();
scheduledExecutorService.execute(new Runnable() {
ByteBuffer fileNameLenBuffer = ByteBuffer.allocate(4);
ByteBuffer fileSizeBuffer = ByteBuffer.allocate(8);
@Override
public void run() {
try {
//读取文件名字长度
int read = socketChannel.read(fileNameLenBuffer);
if (read == -1) {
key.cancel();
socketChannel.close();
return;
}
fileNameLenBuffer.flip();
int fileNameLen = fileNameLenBuffer.getInt();
fileNameLenBuffer.clear();
//读取文件名
ByteBuffer fileNmaeBuffer = ByteBuffer.allocate(fileNameLen);
read = socketChannel.read(fileNmaeBuffer);
if (read == -1) {
key.cancel();
socketChannel.close();
return;
}
fileNmaeBuffer.flip();
byte[] fileNmaeBytes = new byte[fileNmaeBuffer.limit()];
fileNmaeBuffer.get(fileNmaeBytes);
String fileName = new String(fileNmaeBytes);
//读取文件长度
read = socketChannel.read(fileSizeBuffer);
if (read == -1) {
key.cancel();
socketChannel.close();
return;
}
fileSizeBuffer.flip();
long fileSize = fileSizeBuffer.getLong();
fileSizeBuffer.clear();
//获取文件内容
//创建文件目录和文件输出流
File file = new File("E:\\ProjectWorker\\TestJob\\test_cursor\\etc\\" + fileName);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
ByteBuffer dataBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE);;
try (RandomAccessFile fileOutputStream = new RandomAccessFile(file, "rw");
FileChannel fileChannel = fileOutputStream.getChannel()) {
MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, fileSize);
while (map.remaining() > 0) {
while (dataBuffer.remaining() > 0) {
read = socketChannel.read(dataBuffer);
if (read == -1) {
break;
}
}
dataBuffer.flip();
map.put(dataBuffer);
dataBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE);
}
System.out.println(System.currentTimeMillis() + ",File received: " + fileName + ", " + fileSize + " bytes.");
} catch (IOException e) {
e.printStackTrace();
}finally {
dataBuffer.clear();
dataBuffer = null;
}
// 关闭连接
key.cancel();
socketChannel.close();
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
}
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
发送端
package com.chun.zerocopy;
import java.io.File;
import java.io.FileInputStream;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
/**
* @author chun
* @date 2023/4/6 10:04
*/
public class FileSender {
public static void main(String[] args) {
ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(20);
File file = new File("C:\\Users\\Downloads");
File[] files = file.listFiles();
for (File file1 : files) {
if (!file1.isDirectory()) {
scheduledExecutorService.execute(new Thread(()->{
send(file1);
}));
}
}
System.out.println(111);
// System.exit(1);
}
private static void send(File file) {
ByteBuffer allocate = ByteBuffer.allocate(1024);
try {
// 创建一个SocketChannel，连接到接收文件的服务
SocketChannel socketChannel = SocketChannel.open();
socketChannel.connect(new InetSocketAddress("127.0.0.1", 9999));
long sTime = System.currentTimeMillis();
// 获取SocketChannel的输出流
FileInputStream fileInputStream = new FileInputStream(file);
// 获取文件通道
FileChannel fileChannel = fileInputStream.getChannel();
// 使用mmap将文件映射到内存中
// MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size());
// 发送文件名和文件大小
String fileName = file.getName();
byte[] fileNameBytes = fileName.getBytes(StandardCharsets.UTF_8);
long fileSize = file.length();
allocate.putInt(fileNameBytes.length);
allocate.put(fileNameBytes);
allocate.putLong(fileSize);
allocate.flip();
socketChannel.write(allocate);
// 使用零拷贝技术将文件发送给服务端
long sPos = 0;
while (sPos < fileSize) {
sPos += fileChannel.transferTo(sPos, fileSize - sPos, socketChannel);
}
// 关闭流和socket
fileChannel.close();
fileInputStream.close();
socketChannel.close();
//记录时间
System.out.println("fileName="+file.getName() + ",sendTime=" + (System.currentTimeMillis() - sTime));
} catch (Exception e) {
e.printStackTrace();
System.out.println("fileName="+file.getName() + "e=" + e.getMessage());
}
}
}
测试
经测试300M数据0.5秒左右，可以同时传输大量文件