为了比较beansdb，写了一个基于socket的java文件读取测试。
server：
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.apache.commons.io.FileUtils;
import com.taobao.tlog.util.NamedThreadFactory;
public class FileServer {
public void run() throws IOException {
bosses.submit(new Boss());
}
static ExecutorService bosses = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("boss", false));
static ExecutorService workers = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("worker", false));
private static class Boss implements Callable {
@Override
public Object call() throws Exception {
ServerSocket server = new ServerSocket(8080);
while (true) {
Socket socket = server.accept();
workers.submit(new Wroker(socket));
}
}
}
private static class Wroker implements Callable {
Socket socket;
PushbackInputStream input;
public Wroker(Socket socket) throws IOException {
this.socket = socket;
input = new PushbackInputStream(socket.getInputStream(), 1);
}
protected void receive(String path) throws IOException {
File file = new File(path);
byte[] data = FileUtils.readFileToByteArray(file);
OutputStream output = socket.getOutputStream();
output.write(data);
output.flush();
System.out.println(path+" flush");
}
public void accept() throws Exception {
int bytesToRead = 0;
while (true) {
bytesToRead = input.available();
if (bytesToRead > 0) {
byte[] t = new byte[45];
input.read(t);
String path = new String(t);
// System.out.println("path=" + path);
receive(path);
} else {
int b = input.read(); // 此操作会阻塞，直到有数据被读到
if (b < 0) {
throw new IOException(
" end of the socket input stream has been reached,may be server socket is closed!");
} else {
input.unread(b);
continue;
}
}
}
}
@Override
public Object call() {
while (true) {
try {
accept();
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}
}
}
public static void main(String[] args) throws IOException {
new FileServer().run();
}
}
client
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PushbackInputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.text.DecimalFormat;
import java.util.Random;
public class FileClient {
static Random random = new Random();
Socket socket;
PushbackInputStream input;
public FileClient() throws IOException {
init();
}
public void init() throws IOException {
socket = new Socket();
socket.setReuseAddress(true);
socket.setKeepAlive(true);
socket.setTcpNoDelay(true);
socket.setSoLinger(true, 0);
socket.setSoTimeout(60000);
socket.connect(new InetSocketAddress("localhost", 8080), 60000);
input = new PushbackInputStream(socket.getInputStream(), 1);
}
public void send(String path) throws IOException {
// System.out.println(path);
OutputStream out = socket.getOutputStream();
OutputStreamWriter writer = new OutputStreamWriter(out);
writer.write(path);
writer.flush();
}
/**
* System.out.println("readed:" + readBytes + ",bytesToRead:" + bytesToRead);
* <pre>
* readed:-1,bytesToRead:0
* readed:-1,bytesToRead:180224
* readed:131072,bytesToRead:376831
* readed:262144,bytesToRead:344063
* readed:393216,bytesToRead:324300
* readed:524288,bytesToRead:193228
* readed:655360,bytesToRead:62156
* readed:717516,bytesToRead:0
* </pre>
* @throws IOException
*/
public void readLine() throws IOException {
int bytesToRead = 0;
int readBytes = -1;
while (true) {
bytesToRead = input.available();
if (readBytes >= 717516) {
break;
}
if (bytesToRead > 0) {
byte[] data = new byte[bytesToRead];
readBytes += input.read(data);
} else {
int b = input.read(); // 此操作会阻塞，直到有数据被读到
if (b < 0) {
throw new IOException(
" end of the socket input stream has been reached,may be server socket is closed!");
} else {
input.unread(b);
continue;
}
}
}
}
public void test() throws IOException {
DecimalFormat df = new DecimalFormat("#00");
long begin = System.currentTimeMillis();
for (int i = 0; i < 100; i++) {
int id = random.nextInt(10000);
int month = id % 12 + 1;
int day = id % 30 + 1;
int ix = id % 99 + 1;
String file = "/duitang/data/pic_test/2013/" + df.format(month) + "/" + df.format(day) + "/HDFS"
+ df.format(ix) + ".jpeg";
send(file);
readLine();
}
long end = System.currentTimeMillis();
System.out.println((end - begin) + "ms ");
}
public static void main(String[] args) throws IOException {
FileClient client = new FileClient();
for (int i = 0; i < 10; ++i) {
client.test();
}
}
}
测试数据准备，测试数据按照/year/month/day/data 存放，一共36000张图片：
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import org.apache.commons.io.FileUtils;
public class FileUtil {
public static void buildData() throws IOException {
DecimalFormat format = new DecimalFormat("#00");
String path = "/duitang/data/pic_test/2013";
for (int month = 1; month < 13; ++month) {
File mongthPath = new File(path, format.format(month));
mongthPath.mkdirs();
for (int day = 1; day < 31; ++day) {
File dayPath = new File(mongthPath, format.format(day));
dayPath.mkdirs();
for (int i = 1; i < 100; ++i) {
FileUtils.copyFile(new File("/tmp/HDFS.jpeg"), new File(dayPath, "HDFS" + format.format(i)
+ ".jpeg"));
}
}
}
}
}
在我笔记本上的性能：
187ms 174ms 157ms 169ms 171ms 154ms 168ms 155ms 163ms 164ms
产生的图片请求，确保是随机的：
/duitang/data/pic_test/2013/03/27/HDFS48.jpeg flush /duitang/data/pic_test/2013/09/09/HDFS39.jpeg flush /duitang/data/pic_test/2013/02/08/HDFS74.jpeg flush /duitang/data/pic_test/2013/06/12/HDFS87.jpeg flush /duitang/data/pic_test/2013/08/08/HDFS50.jpeg flush /duitang/data/pic_test/2013/01/07/HDFS79.jpeg flush /duitang/data/pic_test/2013/10/16/HDFS52.jpeg flush /duitang/data/pic_test/2013/02/08/HDFS47.jpeg flush /duitang/data/pic_test/2013/07/19/HDFS88.jpeg flush /duitang/data/pic_test/2013/05/29/HDFS14.jpeg flush /duitang/data/pic_test/2013/08/20/HDFS71.jpeg flush /duitang/data/pic_test/2013/08/20/HDFS26.jpeg flush /duitang/data/pic_test/2013/03/09/HDFS45.jpeg flush /duitang/data/pic_test/2013/03/15/HDFS36.jpeg flush /duitang/data/pic_test/2013/09/21/HDFS39.jpeg flush /duitang/data/pic_test/2013/11/23/HDFS53.jpeg flush /duitang/data/pic_test/2013/04/10/HDFS64.jpeg flush /duitang/data/pic_test/2013/02/26/HDFS80.jpeg flush /duitang/data/pic_test/2013/09/27/HDFS06.jpeg flush /duitang/data/pic_test/2013/01/07/HDFS46.jpeg flush /duitang/data/pic_test/2013/04/16/HDFS19.jpeg flush /duitang/data/pic_test/2013/10/10/HDFS88.jpeg flush /duitang/data/pic_test/2013/05/11/HDFS29.jpeg flush /duitang/data/pic_test/2013/08/14/HDFS86.jpeg flush /duitang/data/pic_test/2013/07/07/HDFS85.jpeg flush /duitang/data/pic_test/2013/09/21/HDFS51.jpeg flush
disable linux cache后： 普通FS 326 ms 286 ms 303 ms 292 ms 296 ms 283 ms 292 ms 290 ms 273 ms 289 ms beansdb 368.748903275 ms 326.793909073 ms 337.444067001 ms 322.673082352 ms 336.313009262 ms 331.909179688 ms 329.982995987 ms 275.442838669 ms 271.515130997 ms 333.788871765 ms 基于BufferedRandomAccess 127ms 125ms 125ms 123ms 127ms 126ms 125ms 128ms 127ms 126ms