用lucene3.6搜索数据库和txt文件内容

我们以前经常碰到搜索数据库的内容；用like ％的sql语句；如果数据量大而且多表查询时；速度实在让人难以忍受。。。如果用lucene3.6那就可以把这个恼人的问题解决了。

lucene3.6搜索photo表的title，username，tagname，desr内容；
用一个例题来说明更直观；此例题能搜索中文分词；
（需要mysql5的jdbc包和lucene3.5的包）：

1、数据库我用mysql5；建一个photo表；数据库名是test。
--
-- 表的结构 photo
--
CREATE TABLE IF NOT EXISTS 'photo'(
'photo_id' int(10) unsigned NOT NULL AUTO_INCREMENT,
'title' varchar(11) DEFAULT NULL,
'descr' text,
'user_name' varchar(11) DEFAULT NULL,
'tag_name' varchar(11) DEFAULT NULL,
PRIMARY KEY ('photo_id')
) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=REDUNDANT AUTO_INCREMENT=5 ;
--
-- 导出表中的数据 'photo'
--
INSERT INTO 'photo' ('photo_id', 'title', 'descr', 'user_name', 'tag_name') VALUES
(1, 'z美女j', 'h美女h', 't好人5', 'g美女femal'),
(2, 'l美女k', '5美女', '6美女', 'd美女female'),
(3, 'hagh', '4说的就是我的是f', '', NULL),
(4, 'hagh', '2说的就是我的是g', ' ', NULL);

2、java文件有4个：

文件Photo.java是数据库的photo表的操作文件；内容如下：
package test;
import java.sql.Connection;
import java.util.ArrayList;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
public class Photo {
private long photoId;
private String title;
private String description;
private String userName;
private String tag;
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public long getPhotoId() {
return photoId;
}
public void setPhotoId(long photoId) {
this.photoId = photoId;
}
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public static Photo[] loadPhotos(Connection con) throws Exception {
ArrayList<Photo> list = new ArrayList<Photo>();
PreparedStatement pstm = null;
ResultSet rs = null;
String sql = "select photo_id,title,descr,user_name,tag_name from photo";
try {
pstm = con.prepareStatement(sql);
rs = pstm.executeQuery();
while (rs.next()) {
Photo photo = new Photo();
photo.setPhotoId(rs.getLong(1));
photo.setTitle(rs.getString(2));
photo.setDescription(rs.getString(3));
photo.setUserName(rs.getString(4));
photo.setTag(rs.getString(5));
list.add(photo);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
if (rs != null) {
rs.close();
}
if (pstm != null) {
pstm.close();
}
}
return (Photo[]) list.toArray(new Photo[list.size()]);
}
}

文件IndexerFile.java是把数据库的内容备份成索引文件到磁盘中去；
内容如下：
package test;
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.lucene.document.Field;
public class IndexerFile {
public static int indexFile(String indexDir,Photo[] list) throws IOException{
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36));
conf.setOpenMode(OpenMode.CREATE);
IndexWriter writer = new IndexWriter(FSDirectory.open(new File(indexDir)), conf);

for(int i=0;i<list.length;i++){
Document doc=new Document();
doc.add(new Field("photoId", String.valueOf(list[i].getPhotoId()), Field.Store.YES, Field.Index.NO));
if(list[i].getTitle()!=null && list[i].getTitle().length()>0)
doc.add(new Field("title", list[i].getTitle(), Field.Store.YES, Field.Index.ANALYZED));
if(list[i].getDescription()!=null && list[i].getDescription().length()>0)
doc.add(new Field("description", list[i].getDescription(), Field.Store.YES, Field.Index.ANALYZED));
if(list[i].getUserName()!= null && list[i].getUserName().length()>0)
doc.add(new Field("userName", list[i].getUserName(), Field.Store.YES, Field.Index.ANALYZED));
if(list[i].getTag()!= null && list[i].getTag().length()>0)
doc.add(new Field("tag", list[i].getTag(), Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}

int numIndexed = writer.maxDoc();
writer.forceMerge(1);
writer.close();
return numIndexed;
}
}

文件SearcherFile.java是搜索磁盘索引文件内容的；
内容如下：
package test;
import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Version;
public class SearcherFile {
public static void search(IndexSearcher searcher, String[] q) throws IOException, ParseException {
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_36);
String[] fields = {"title","description","tag","userName"};
Query query = MultiFieldQueryParser.parse(Version.LUCENE_36, q, fields, analyzer);
TopDocs topDocs = searcher.search(query, 100);//100是显示队列的Size
ScoreDoc[] hits = topDocs.scoreDocs;
System.out.println("共有" + searcher.maxDoc() + "条索引，命中" + hits.length + "条");
for (int i = 0; i < hits.length; i++) {
int DocId = hits[i].doc;
Document document = searcher.doc(DocId);
System.out.println("photoId==="+document.get("photoId"));
}
}
}

文件TestDb.java是操作的主文件；
内容如下：
package test;
import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Date;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
public class TestDb {
public final static String indexDir ="E:\\TestLucene";
private static Connection getConnection() {
Connection conn = null;
String url = "jdbc:mysql://localhost:3306/test";
String userName = "root";
String password = "root";
try {
Class.forName("com.mysql.jdbc.Driver");
conn = java.sql.DriverManager
.getConnection(url, userName, password);
} catch (Exception e) {
e.printStackTrace();
System.out.println("Error Trace in getConnection() : "
+ e.getMessage());
}
return conn;
}
public static void main(String[] args) throws IOException, ParseException, SQLException {
index();//做索引
IndexSearcher searcher=null;
try{
IndexReader reader = IndexReader.open(FSDirectory.open(new File(indexDir)));
searcher = new IndexSearcher(reader);
search(searcher);//搜索
}catch(Exception e){
e.printStackTrace();
}finally{
if(searcher!=null)
searcher.close();
}
}
public static void search(IndexSearcher searcher) throws IOException, ParseException{
//以下是搜索的关键词
String[] q = {"美女1","美女2","好人3","好人5"};
long start=new Date().getTime();
SearcherFile.search(searcher,q);
long end=new Date().getTime();
System.out.println("花费时间："+(double)(end-start)/1000+"秒");
}
public static void index() throws SQLException{
Connection conn = null;
try {
conn = getConnection();
Photo[] list = Photo.loadPhotos(conn);
IndexerFile.indexFile(indexDir,list);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (conn != null) {
conn.close();
}
}
}
}

二、下面是lucene3.5搜索txt文本文件

建一个E:\\TestLucene\\fileS的文件夹,放需要搜索的文件。
在该文件夹里面随便建三个txt文件，"1.txt","2.txt"和"3.txt"



其中1.txt的内容如下：

老周
北京人民
2009年

2.txt和3.txt也随便写些。



再建一个E:\\TestLucene\\fileIndex的文件夹；放索引文件。





java文件TestQueryFile：内容如下



package test;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.apache.lucene.document.Field;
public class TestQueryFile {

public static void main(String[] args) throws Exception {
indexF();
String queryString = "北京";
Query query = null;
IndexReader reader = IndexReader.open(FSDirectory.open(new File("E:\\TestLucene\\fileIndex")));
IndexSearcher searcher = new IndexSearcher(reader);
String fields = "body";
try {
QueryParser qp = new QueryParser(Version.LUCENE_36, fields, new StandardAnalyzer(Version.LUCENE_36));//有变化的地方
query = qp.parse(queryString);
} catch (ParseException e) {
}
if (searcher != null) {
TopDocs topDocs = searcher.search(query, 100);//100是显示队列的Size
ScoreDoc[] hits = topDocs.scoreDocs;
System.out.println("共有" + searcher.maxDoc() + "条索引，命中" + hits.length + "条");
}
}

private static void indexF() throws Exception {

File fileDir = new File("E:\\TestLucene\\fileS");


File indexDir = new File("E:\\TestLucene\\fileIndex");

IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, new StandardAnalyzer(Version.LUCENE_36));
conf.setOpenMode(OpenMode.CREATE);
IndexWriter indexWriter = new IndexWriter(FSDirectory.open(indexDir), conf);

File[] textFiles = fileDir.listFiles();
long startTime = new Date().getTime();

//增加document到索引去
for (int i = 0; i < textFiles.length; i++) {
if (textFiles[i].isFile()
&& textFiles[i].getName().endsWith(".txt")) {
System.out.println("File " + textFiles[i].getCanonicalPath()
+ "正在被索引....");
String temp = FileReaderAll(textFiles[i].getCanonicalPath(),
"GBK");
System.out.println(temp);
Document document = new Document();
Field FieldPath = new Field("path", textFiles[i].getPath(),
Field.Store.YES, Field.Index.NO);
Field FieldBody = new Field("body", temp, Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
document.add(FieldPath);
document.add(FieldBody);
indexWriter.addDocument(document);
}
}
//optimize()方法是对索引进行优化
indexWriter.forceMerge(1);
indexWriter.close();

//测试一下索引的时间
long endTime = new Date().getTime();
System.out
.println("这花费了"
+ (endTime - startTime)
+ " 毫秒来把文档增加到索引里面去!"
+ fileDir.getPath());
}

private static String FileReaderAll(String FileName, String charset)
throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(
new FileInputStream(FileName), charset));
String line = null;
StringBuffer temp = new StringBuffer("");

while ((line = reader.readLine()) != null) {
temp.append(line);
}
reader.close();
return temp.toString();
}
}

一执行就知道结果了