solr build索引性能

测试代码

package mytest;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.CoreContainer;

public class SolrTest {

	public static void test1() throws Exception {
		HttpSolrServer solrServer = new HttpSolrServer("http://localhost:8080/solr/dtrace");
		solrServer.setRequestWriter(new BinaryRequestWriter());
		int size = 100;

		long begin = System.currentTimeMillis();
		for (int i = 0; i < size; ++i) {
			SolrInputDocument doc1 = new SolrInputDocument();
			doc1.addField("id", i);
			doc1.addField("rowkey", "100 p");
			doc1.addField("cf", "[{30:50}]");
			doc1.addField("timestamp", System.currentTimeMillis());
			solrServer.add(doc1);
		}
		long end = System.currentTimeMillis();
		System.out.println(" add cost:" + (end - begin) + "ms");
		begin = System.currentTimeMillis();
		solrServer.commit();
		end = System.currentTimeMillis();
//		System.out.println(" commit " + size + " cost:" + (end - begin) + " ms");
	}

	static ExecutorService service = Executors.newFixedThreadPool(20);

	static CoreContainer container = new CoreContainer("/duitang/data/solr");
	static {
		container.load();
	}
	static EmbeddedSolrServer solrServer = new EmbeddedSolrServer(container, "dtrace");

	public static void _test2(int round) throws Exception {
		int count = 10000;
		int size = count * round;
		final CountDownLatch latch = new CountDownLatch(count);
//		final HttpSolrServer solrServer = new HttpSolrServer("http://localhost:8080/solr/dtrace");
//		solrServer.setRequestWriter(new BinaryRequestWriter());

		long begin = System.currentTimeMillis();

		for (int i = size - count; i < size; ++i) {
			final int id = i;
			service.submit(new Runnable() {

				public void run() {
					SolrInputDocument doc1 = new SolrInputDocument();
					doc1.addField("id", id);
					doc1.addField("rowkey",
							"12345678901234567890 12345678901234567890 12345678901234567890 12345678901234567890 12345678901234567890");
					doc1.addField(
							"cf",
							"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890");
					doc1.addField("timestamp", System.currentTimeMillis());
					try {
						solrServer.add(doc1);
						latch.countDown();
					} catch (Exception e) {
						e.printStackTrace();
					}
				}
			});
		}
		latch.await();
		long end = System.currentTimeMillis();
		System.out.println(" add[" + size + "] " + count + " cost:" + (end - begin) + "ms");

//		begin = System.currentTimeMillis();
//		solrServer.commit(false, false, true);
//		solrServer.commit();
//		end = System.currentTimeMillis();
//		System.out.println(" commit " + size + " cost:" + (end - begin) + "ms");
//		service.shutdown();
	}

	public static void test2() throws Exception {
		ExecutorService service = Executors.newFixedThreadPool(1);
		int count = 10000;
		final CountDownLatch latch = new CountDownLatch(count);
		long begin = System.currentTimeMillis();

		for (int i = 7342; i <= count; ++i) {
			final int index = i;
			service.submit(new Runnable() {

				public void run() {
					try {
						_test2(index);
						latch.countDown();
					} catch (Exception e) {
						e.printStackTrace();
					}

				}
			});
		}
		latch.await();
		long end = System.currentTimeMillis();
		System.out.println(" add finish " + count + " cost:" + (end - begin) + "ms");
	}

	public static void test3() throws Exception {

//		CoreContainer container = new CoreContainer("/duitang/data/solr");
//		container.load();
//		EmbeddedSolrServer solrServer = new EmbeddedSolrServer(container, "dtrace");
		SolrInputDocument doc1 = new SolrInputDocument();
		doc1.addField("id", 1);
		doc1.addField("rowkey", "100 p");
		doc1.addField("cf", "[{30:50}]");
		doc1.addField("timestamp", System.currentTimeMillis());
		solrServer.add(doc1);
//		solrServer.commit();
		System.out.println("ok");
	}

	public static void close() {
		solrServer.shutdown();
	}

	public static void main(String[] args) throws Exception {
		test2();
//		close();
//		RAMDirectory rdir = new RAMDirectory();
//		String fileList[] = rdir.listAll();
//		for (int i = 0; i < fileList.length; i++) {
//			
//		}

//		FileSystem fs = FileSystem.get(null);
//		 fs.startLocalOutput(fsOutputFile, tmpLocalFile)
//		FSDirectory.open(path)
		//IndexWriter indexWriter = new IndexWriter();
	}
}

 每次build数据，1k字符串。

 

 

一.测试add性能

1.单个add

平均cost:4ms

 

2. 顺序执行，add 100个doc:

add cost:564ms

add cost:404ms

add cost:349ms

add cost:393ms

add cost:368ms

 

3. 20个并发，add 100个doc(增加到30，40个并发，性能没有增加):

add cost:74ms

add cost:78ms

add cost:61ms

add cost:60ms

 

3.20个并发， 不同doc的耗时

add 1000 cost:367ms

add 5000 cost:812ms

add 10000 cost:1624ms

add 20000 cost:3190ms

add 100000 cost:16874ms

 

二. 测试commit性能

1.等到多少条doc，提交一次的耗时

commit 100   cost:600 ms

commit 1000 cost:1053ms

commit 5000 cost:2130ms

commit 10000 cost:2542ms

commit 20000 cost:3057ms

commit 30000 cost:3036ms

commit 40000 cost:3408ms

commit 100000 cost:4808ms

 

三. 并发commit测试：

当开启5个线程并发commit时候，load上升到10+,会抛出异常：

org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error opening new searcher. exceeded limit of maxWarmingSearchers=2, try again later.

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:491)

at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:197)

at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)

at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)

at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)

at mytest.SolrTest.test2(SolrTest.java:74)

at mytest.SolrTest$2.run(SolrTest.java:88)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)

at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)

at java.util.concurrent.FutureTask.run(FutureTask.java:138)

at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)

at java.lang.Thread.run(Thread.java:662)

 

 

三. 网络连接

测试发现solr不会开启长连接。

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46166         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46197         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46243         TIME_WAIT   -               

tcp6       0    183 127.0.0.1:8080          127.0.0.1:46302         ESTABLISHED 20112/java      

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46173         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46228         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46230         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46278         TIME_WAIT   -               

tcp6       0    187 127.0.0.1:8080          127.0.0.1:46298         ESTABLISHED 20112/java      

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46158         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:46299         127.0.0.1:8080          ESTABLISHED 3321/java       

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46137         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46160         TIME_WAIT   -               

tcp6       0      0 127.0.0.1:8080          127.0.0.1:46134         TIME_WAIT   -               

tcp6       0    187 127.0.0.1:8080          127.0.0.1:46288         ESTABLISHED 20112/java      

yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ netstat -antp | grep 8080 | wc -l

331

yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ netstat -antp | grep 8080 | wc -l

411

yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ netstat -antp | grep 8080 | wc -l

660

yunpeng@yunpeng-duitang:/duitang/data/solr$ netstat -antp | grep 8080 | wc -l

10018

 

 

四. 索引数据量

插入30万条数据，单条数据1k，占用磁盘空间：9.5M,30万条1K数据实际应占用300MB，可见luence的索引是经过压缩的。大概是30:1。 （结论不正确，估计是因为测试是大量重复数据有关）

 

五.测试solr各种commit不同配置下的性能。

1.配置1

<updateHandler class="solr.DirectUpdateHandler2">
    <autoCommit>
     <maxDocs>10000</maxDocs>
     <maxTime>1000</maxTime>
    <openSearcher>false</openSearcher>
    </autoCommit>
      <updateLog>
           <str name="dir">${solr.shard.data.dir:}</str>
      </updateLog>
 </updateHandler>

结果：

1.无法查询到结果。

2.yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ du -sh data/*

2.9M data/index

39M data/tlog

3. add 5000 cost:2405ms

 

2. 配置2：

<updateHandler class="solr.DirectUpdateHandler2">
    <autoCommit>
     <maxDocs>10000</maxDocs>
     <maxTime>1000</maxTime>
    <openSearcher>true</openSearcher>
    </autoCommit>
      <updateLog>
           <str name="dir">${solr.shard.data.dir:}</str>
      </updateLog>
 </updateHandler>

结果：

1.可以查询到结果。

2.yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ du -sh data/*

1.7M data/index

27M data/tlog

3. add5000 cost:2726ms

 

3. 配置3：

  <updateHandler class="solr.DirectUpdateHandler2">
        <autoCommit>
                 <maxTime>2000</maxTime>
                <openSearcher>false</openSearcher>
        </autoCommit>
        <autoSoftCommit>
                <maxTime>10000</maxTime>
        </autoSoftCommit>
          <updateLog>
               <str name="dir">${solr.shard.data.dir:}</str>
          </updateLog>
     </updateHandler>

结果：

1.可以查询到结果。

2.yunpeng@yunpeng-duitang:/duitang/data/solr/dtrace$ du -sh data/*

2.0M data/index

31M data/tlog

3. add 5000 cost:1264ms

 

五.结论

1. 通过http api solr单机build索引最大QPS 5k。