SolrJ 操作HttpSolrServer, ConcurrentUpdateSolrServer和CloudSolrServer
HttpSolrServer 使用了Apache Commons HTTP客户端来连接Solr. 注意在Solr 4.x中， CommonsHttpSolrServer已经改变为HttpSolrServer以及StreamingUpdateSolrServer已经改变为ConcurrentUpdateSolrServer 。ConcurrentUpdateSolrServer更适合update 操作，而HttpSolrServer 更适合query操作。
添加document或是修改document。假如这个document已经存在，就会update这个document。代码片段如下:
public void indexDocs() throws IOException, SolrServerException {
server.setParser(new XMLResponseParser());
//Adds the docs and commit them.
Collection<SolrInputDocument> docs = new LinkedList<SolrInputDocument>();
/* i is used as identification of a document, which is treated as unique key.*/
SolrInputDocument doc2 ;
/*一千条数据，花费大约一小时，生产660M。使用多线程并发执行估计更好的*/
for(int i =10000000; i < 10000002; i++){
doc2 = new SolrInputDocument();
doc2.addField("customer_id", i);
doc2.addField("name", "John Natch-" + i);
doc2.addField("level", "VIP");
doc2.addField("sex", "男");
doc2.addField("address", "【【【【【金刚金刚金刚金刚金刚金】】】】" + i);
System.out.println("add doc "+ i);
docs.add(doc2);
if(docs.size() == 1000){
server.add(docs);
server.commit();
logger.info("commit 1000 doc "+ i);
docs.clear();
}
/*
To immediately commit after adding documents, you could use:
UpdateRequest req = new UpdateRequest();
req.setAction( UpdateRequest.ACTION.COMMIT, false, false );
req.add( docs );
UpdateResponse rsp = req.process( server );
*/
}
server.add(docs);
server.commit();
logger.info("Commits successfully!......");
}
能够执行代码前，在Solr core的配置文件shema.xml中配置具体的字段。
<!-- core 'customer' schema field definition --> <field name="customer_id" type="int" indexed="true" stored="true" required="true" multiValued="false"/> <field name="name" type="string" indexed="true" stored="true"/> <field name="sex" type="string" indexed="true" stored="false"/> <field name="level" type="string" indexed="true" stored="true"/> <field name="address" type="string" indexed="true" multiValued="true" stored="true"/>
<uniqueKey>customer_id</uniqueKey>
删除操作：
private void commitDocs(Collection<SolrInputDocument> docs){
try {
//server.deleteById(1) //specify the id list you want to be deleted.
server.add(docs);
server.commit();
docs.clear();
} catch (SolrServerException e) {
logger.error("SolrServerException", e);
} catch (IOException e) {
logger.error("IOException", e) ;
}
}
与数据集成，实现使用SolrJ操作数据库。当然，这个可以使用Solr DIH实现。两种各有其优缺点，根据实际的应用来选择具体的实现方式。
public void indexDocsWithDB(){
PoolingDataSourceDemo dataSource = new PoolingDataSourceDemo();
List<List<Object>> rows = dataSource.executeQuerySQL("select * from customer");
String[] columnNames = dataSource.getColNames();
Collection<SolrInputDocument> docs = new LinkedList<SolrInputDocument>();
SolrInputDocument doc ;
for(List row : rows) {
int size = row.size() + 1;
doc = new SolrInputDocument();
for(int i = 1; i < size ; i++){
doc.addField(columnNames[i], row.get(i-1)) ;
logger.info(columnNames[i]+"add filed "+ row.get(i-1)) ;
}
docs.add(doc);
if(docs.size() > 100){
commitDocs(docs);
}
}
if(docs.size() > 0){
commitDocs(docs);
}
} 完整的代码：
PoolingDataSourceDemo.java 实现线程池连接数据库。
import net.spy.memcached.compat.log.Logger;
import net.spy.memcached.compat.log.LoggerFactory;
import org.apache.commons.dbcp.*;
import org.apache.commons.pool.impl.GenericObjectPool;
import javax.sql.DataSource;
import java.sql.*;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
/**
* @author John Liu
* @see
*/
public class PoolingDataSourceDemo {
private final static Logger logger = LoggerFactory.getLogger(PoolingDataSourceDemo.class) ;
/* These properties can be configured in a properties type file*/
private final static String CONNECTION_URL = "jdbc:mysql://localhost/pythondb?autoReconnect=true";
private final static String DRIVER_CLASS = "com.mysql.jdbc.Driver";
private final static String USER_NAME = "elite";
private final static String PASSWORD = "elite";
private final static int MAX_ACTIVE_NUMBER = 10;
private static GenericObjectPool connectionPool = null;
private String[] colNames ;
private static DataSource dataSource;
static {
dataSource = initDataSource();
}
public GenericObjectPool getConnectionPool() {
return connectionPool;
}
public List<List<Object>> executeQuerySQL(String querySQL){
Connection conn = null;
Statement stmt = null;
ResultSet resultSet = null;
List<List<Object>> result = new LinkedList<List<Object>>();
try {
logger.info("Creating connection.");
conn = dataSource.getConnection();
stmt = conn.createStatement();
resultSet = stmt.executeQuery(querySQL);
//show the connection pool status
printDataSourceStats();
logger.info("Results:");
int columnCount = resultSet.getMetaData().getColumnCount();
ResultSetMetaData rsm = resultSet.getMetaData();
colNames = new String[columnCount + 1];
for (int i = 1; i < (columnCount + 1); i++) {
colNames[i] = rsm.getColumnName(i).toLowerCase();
logger.info("column name: "+ colNames[i]) ;
}
List<Object> list ;
while(resultSet.next()) {
list = new ArrayList<Object>() ;
for(int i=1; i<= columnCount; i++) {
Object obj = getColumnValue(rsm, resultSet, colNames, i);
list.add(obj) ;
}
result.add(list);
}
} catch(SQLException e) {
e.printStackTrace();
shutdownDataSource(dataSource);
} finally {
try { if (resultSet != null) resultSet.close(); } catch(Exception e) { }
try { if (stmt != null) stmt.close(); } catch(Exception e) { }
try { if (conn != null) conn.close(); } catch(Exception e) { }
logger.info("result size: "+ result.size());
return result;
}
}
public Object getColumnValue(ResultSetMetaData rsm, ResultSet rs, String[] colNames, int j) throws SQLException {
Object f = null;
if (colNames[j] != null) {
switch (rsm.getColumnType(j)){
case Types.BIGINT:{
f = rs.getLong(j);
break;
}
case Types.INTEGER: {
f = rs.getInt(j);
break;
}
case Types.DATE:{
f = rs.getDate(j);
break;
}
case Types.FLOAT:{
f = rs.getFloat(j);
break;
}
case Types.DOUBLE:{
f = rs.getDouble(j);
break;
}
case Types.TIME: {
f = rs.getDate(j);
break;
}
case Types.BOOLEAN:{
f = rs.getBoolean(j);
break;
}
default:{
f = rs.getString(j);
}
}
}
logger.info("column value: "+ f) ;
return f;
}
/**
* [mysql]
* #hibernate.connection.driver_class com.mysql.jdbc.Driver
#hibernate.connection.url jdbc:mysql:///test
#hibernate.connection.username gavin
#hibernate.connection.password
* @return DataSource
*/
public static DataSource initDataSource(){
//
// Load JDBC Driver class.
//
try {
Class.forName(DRIVER_CLASS).newInstance();
} catch (InstantiationException e) {
logger.error("InstantiationException error", e);
} catch (IllegalAccessException e) {
logger.error("IllegalAccessException error", e);
} catch (ClassNotFoundException e) {
logger.error("ClassNotFoundException error", e);
}
//
// Creates an instance of GenericObjectPool that holds our
// pool of connections object.
//
connectionPool = new GenericObjectPool();
connectionPool.setMaxActive(MAX_ACTIVE_NUMBER);
//
// Creates a connection factory object which will be use by
// the pool to create the connection object. We passes the
// JDBC url info, username and password.
//
ConnectionFactory cf = new DriverManagerConnectionFactory(
CONNECTION_URL,
USER_NAME,
PASSWORD);
//
// Creates a PoolableConnectionFactory that will wraps the
// connection object created by the ConnectionFactory to add
// object pooling functionality.
//
PoolableConnectionFactory pcf =
new PoolableConnectionFactory(cf, connectionPool,
null, null, false, true);
return new PoolingDataSource(connectionPool);
}
public void printDataSourceStats() {
logger.info("Max : " + getConnectionPool().getMaxActive() + "; " +
"Active: " + getConnectionPool().getNumActive() + "; " +
"Idle : " + getConnectionPool().getNumIdle());
}
public void shutdownDataSource(DataSource ds) throws SQLException {
BasicDataSource bds = (BasicDataSource) ds;
bds.close();
}
public String[] getColNames() {
return colNames;
}
public void setColNames(String[] colNames) {
this.colNames = colNames;
}
}
SolrIndex.java 实现SolrJ的CRUD操作。
public class SolrIndex {
Logger logger = LoggerFactory.getLogger(SolrIndex.class) ;
/*specified the core customer url*/
private static final String CORE_CUSTOMER_URL= "http://localhost:8088/solr/customer";
private static HttpSolrServer server;
static {
server = new HttpSolrServer(CORE_CUSTOMER_URL);
server.setMaxRetries(1); // defaults to 0. > 1 not recommended.
server.setConnectionTimeout(5000); // 5 seconds to establish TCP
// Setting the XML response parser is only required for cross
// version compatibility and only when one side is 1.4.1 or
// earlier and the other side is 3.1 or later.
server.setParser(new XMLResponseParser()); // binary parser is used by default
// The following settings are provided here for completeness.
// They will not normally be required, and should only be used
// after consulting javadocs to know whether they are truly required.
server.setSoTimeout(1000); // socket read timeout
server.setDefaultMaxConnectionsPerHost(1000);
server.setMaxTotalConnections(1000);
server.setFollowRedirects(false);
// defaults to false
// allowCompression defaults to false.
// Server side must support gzip or deflate for this to have any effect.
server.setAllowCompression(true);
}
/**
* Index a document with specified fields in doc.
* @throws IOException
* @throws SolrServerException
*/
public void indexDocs() throws IOException, SolrServerException {
server.setParser(new XMLResponseParser());
//Adds the docs and commit them.
Collection<SolrInputDocument> docs = new LinkedList<SolrInputDocument>();
/* i is used as identification of a document, which is treated as unique key.*/
SolrInputDocument doc2 ;
/*一千条数据，花费大约一小时，生产660M。使用多线程并发执行估计更好的*/
for(int i =10000000; i < 10000002; i++){
doc2 = new SolrInputDocument();
doc2.addField("customer_id", i);
doc2.addField("name", "John Natch-" + i);
doc2.addField("level", "VIP");
doc2.addField("sex", "男");
doc2.addField("address", "【【【【【金刚金刚金刚金刚金刚金】】】】" + i);
System.out.println("add doc "+ i);
docs.add(doc2);
if(docs.size() == 1000){
server.add(docs);
server.commit();
logger.info("commit 1000 doc "+ i);
docs.clear();
}
/*
To immediately commit after adding documents, you could use:
UpdateRequest req = new UpdateRequest();
req.setAction( UpdateRequest.ACTION.COMMIT, false, false );
req.add( docs );
UpdateResponse rsp = req.process( server );
*/
}
server.add(docs);
server.commit();
logger.info("Commits successfully!......");
}
/**
* solrJ与 database 集成，对数据库中的数据建立索引。当然，这个可以使用Solr DIH取代。
*/
public void indexDocsWithDB(){
PoolingDataSourceDemo dataSource = new PoolingDataSourceDemo();
List<List<Object>> rows = dataSource.executeQuerySQL("select * from customer");
String[] columnNames = dataSource.getColNames();
Collection<SolrInputDocument> docs = new LinkedList<SolrInputDocument>();
SolrInputDocument doc ;
for(List row : rows) {
int size = row.size() + 1;
doc = new SolrInputDocument();
for(int i = 1; i < size ; i++){
doc.addField(columnNames[i], row.get(i-1)) ;
logger.info(columnNames[i]+"add filed "+ row.get(i-1)) ;
}
docs.add(doc);
if(docs.size() > 100){
commitDocs(docs);
}
}
if(docs.size() > 0){
commitDocs(docs);
}
}
private void commitDocs(Collection<SolrInputDocument> docs){
try {
//server.deleteById(1) //specify the id list you want to be deleted.
server.add(docs);
server.commit();
docs.clear();
} catch (SolrServerException e) {
logger.error("SolrServerException", e);
} catch (IOException e) {
logger.error("IOException", e) ;
}
}
/**
* Query documents with specified query value.
* @throws SolrServerException
*/
public void queryDocs() throws SolrServerException {
HttpSolrServer server = new HttpSolrServer(CORE_CUSTOMER_URL );
server.setParser(new XMLResponseParser());
/*query statement settings*/
SolrQuery query = new SolrQuery();
query.setQuery("李玲");
query.setStart(0);
query.setRows(10);
QueryResponse response = server.query( query );
SolrDocumentList documents = response.getResults();
Iterator<SolrDocument> itr = documents.iterator();
logger.info("id \t name");
while (itr.hasNext()) {
SolrDocument doc = itr.next();
logger.info(doc.getFieldValue("customer_id") + ":" + "\t"+doc.
getFieldValue("name"));
}
}
public void delete(){
try {
server.deleteByQuery( "*:*" );
server.commit();
} catch (SolrServerException e) {
logger.error("SolrServerException", e);
} catch (IOException e) {
logger.error("IOException", e);
}
}
public static void main(String[] args){
SolrIndex indexer = new SolrIndex();
long startTime = System.currentTimeMillis();
/*do index with specified documents*/
try {
indexer.indexDocs();
} catch (IOException e) {
e.printStackTrace();
} catch (SolrServerException e) {
e.printStackTrace();
}
// try {
// indexer.queryDocs();
// } catch (SolrServerException e) {
// e.printStackTrace();
// }
/*integration with db. It takes 1214 ms*/
// indexer.delete();
// indexer.indexDocsWithDB();
System.out.println("--------It takes "+ (System.currentTimeMillis() - startTime) + " ms");
}
}
另外，SolrJ操作Solr Cloud的机制与HttpSolrServer一样，除了Http的设置使用CloudSolrServer意外。
CloudSolrServer server = new CloudSolrServer("localhost:9983");
server.setDefaultCollection("collection1");
SolrInputDocument doc = new SolrInputDocument();
doc.addField( "id", "1234");
doc.addField( "name", "A lovely summer holiday");
server.add(doc);
server.commit();
运行代码前，假如下列依赖
