二十七、Luence在项目中运用
public class LuceneContext {
private static LuceneContext instance;
private static final String INDEX_PATH = "d:/lucene/test";
private static IndexWriter writer;
private static Analyzer analyzer;
private static Version version;
private static NRTManager nrtMgr;
private static SearcherManager mgr;
private static Directory directory;
private LuceneContext(){}
public static LuceneContext getInstance() {
if(instance==null){
System.out.println("init");
init();
instance = new LuceneContext();
}
return instance;
}
private static void init() {
try {
directory = FSDirectory.open(new File(INDEX_PATH));
version = Version.LUCENE_35;
String dicUrl = LuceneContext.class.getClassLoader().getResource("data").getPath();
analyzer = new MMSegAnalyzer(dicUrl);
writer = new IndexWriter(directory,new IndexWriterConfig(version,analyzer));
nrtMgr = new NRTManager(writer, new SearcherWarmer() {
@Override
public void warm(IndexSearcher arg0) throws IOException {
System.out.println("reopen index");
}
});
mgr = nrtMgr.getSearcherManager(true);
NRTManagerReopenThread reopenThread = new NRTManagerReopenThread(nrtMgr, 5.0,0.025);
reopenThread.setName("NRTManager reopen thread");
reopenThread.setDaemon(true);
reopenThread.start();
} catch (IOException e) {
e.printStackTrace();
}
}
public IndexSearcher getSearcher() {
return mgr.acquire();
}
public void releaseSearcher(IndexSearcher searcher) {
try {
mgr.release(searcher);
} catch (IOException e) {
e.printStackTrace();
}
}
public void commitIndex() {
try {
writer.commit();
writer.forceMerge(3);
} catch (CorruptIndexException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public NRTManager getNRTManager() {
return nrtMgr;
}
public Version getVersion() {
return version;
}
public Analyzer getAnalyzer() {
return analyzer;
}
}
public class IndexField {
private String id;
private String title;
private List<String> content;
private List<String> atths;
private int parentId;
private int objId;
private Date createDate;
private String type;
public class Index {
private int msgId;
private String title;
private String summary;
private Date createDate;
@Entity
@Table(name="temp_index")
public class TempIndex {
private int id;
private int objId;
private String type;//Message,Attachment
private int operator;
@Service("indexService")
public class IndexService implements IIndexService {
private Document field2Doc(IndexField field) {
Document doc = new Document();
doc.add(new Field("id",field.getId(),Field.Store.YES,Field.Index.NOT_ANALYZED_NO_NORMS));
doc.add(new Field("title",field.getTitle(),Field.Store.YES,Field.Index.ANALYZED));
for(String content:field.getContent()) {
doc.add(new Field("content",content,Field.Store.NO,Field.Index.ANALYZED));
}
if(field.getAtths()!=null) {
for(String att:field.getAtths()) {
doc.add(new Field("atts",att,Field.Store.YES,Field.Index.NO));
}
}
doc.add(new NumericField("objId",Field.Store.YES,true).setIntValue(field.getObjId()));
doc.add(new NumericField("createDate",Field.Store.YES,true).setLongValue(field.getCreateDate().getTime()));
return doc;
}
@Override
public void addIndex(IndexField fields,boolean inDatabase) {
try {
if(inDatabase) {
TempIndex ti = new TempIndex();
ti.setAdd();
ti.setObjId(fields.getObjId());
ti.setType(fields.getType());
tempIndexDao.add(ti);
}
NRTManager nrtMgr = LuceneContext.getInstance().getNRTManager();
Document doc = field2Doc(fields);
nrtMgr.addDocument(doc);
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void deleteIndex(String id,String type) {
try {
TempIndex ti = new TempIndex();
ti.setDelete();
///xx_xx
ti.setId(Integer.parseInt(id));
ti.setType(type);
tempIndexDao.add(ti);
LuceneContext.getInstance().getNRTManager().deleteDocuments(new Term("id",id));
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void updateIndex(IndexField fields) {
try {
TempIndex ti = new TempIndex();
ti.setDelete();
///xx_xx
ti.setId(fields.getObjId());
ti.setType(fields.getType());
tempIndexDao.add(ti);
NRTManager nrtMgr = LuceneContext.getInstance().getNRTManager();
Document doc = field2Doc(fields);
nrtMgr.updateDocument(new Term("id",fields.getId()), doc);
} catch (IOException e) {
e.printStackTrace();
}
}
private String highligher(String text,Query query,String field) {
try {
QueryScorer scorer = new QueryScorer(query);
Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
Formatter formatter = new SimpleHTMLFormatter("<span class='lighter'>","</span>");
Highlighter lighter = new Highlighter(formatter,scorer);
lighter.setTextFragmenter(fragmenter);
String ht = lighter.getBestFragment(LuceneContext.getInstance().getAnalyzer(),
field,text);
if(ht==null) {
if(text.length()>=200) {
text = text.substring(0, 200);
text=text+"....";
}
return text;
}
else return ht.trim();
} catch (IOException e) {
e.printStackTrace();
} catch (InvalidTokenOffsetsException e) {
e.printStackTrace();
}
return text;
}
private ScoreDoc getLastDoc(int pageOffset,IndexSearcher searcher,Query query) {
if(pageOffset<=0) return null;
try {
TopDocs tds = searcher.search(query,pageOffset-1);
return tds.scoreDocs[pageOffset-1];
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
@Override
public PageObject<Index> findByIndex(String condition) {
if(condition==null) condition = "";
IndexSearcher searcher = LuceneContext.getInstance().getSearcher();
PageObject<Index> pages = new PageObject<Index>();
List<Index> datas = new ArrayList<Index>();
try {
int pageSize = SystemContext.getPageSize();
int pageOffset = SystemContext.getPageOffset();
MultiFieldQueryParser parser = new MultiFieldQueryParser(LuceneContext.getInstance().getVersion(),
new String[]{"title","content"}, LuceneContext.getInstance().getAnalyzer());
Query query = parser.parse(condition);
TopDocs tds = searcher.searchAfter(getLastDoc(pageOffset,searcher,query),
query, pageSize);
int totalRecord = tds.totalHits;
List<Integer> msgs = new ArrayList<Integer>();
for(ScoreDoc sd:tds.scoreDocs) {
Document doc = searcher.doc(sd.doc);
Index index = new Index();
index.setCreateDate(new Date(Long.parseLong(doc.get("createDate"))));
String title = doc.get("title");
index.setTitle(highligher(title,query,"title"));
String[] ans = doc.getValues("atts");
StringBuffer content = new StringBuffer();
if(ans!=null) {
for(String fn:ans) {
content.append(IndexUtil.file2String(fn));
}
}
index.setSummary(content.toString());
int msgId = Integer.parseInt(doc.get("id"));
index.setMsgId(msgId);
msgs.add(msgId);
datas.add(index);
}
Map<Integer,String> contents = messageDao.listMessageContent(msgs);
for(int i=0;i<datas.size();i++) {
Index index = datas.get(i);
String content = contents.get(index.getMsgId());
content = content+index.getSummary();
datas.get(i).setSummary(highligher(content,query,"content"));
}
pages.setDatas(datas);
pages.setOffset(pageOffset);
pages.setPageSize(pageSize);
pages.setTotalRecord(totalRecord);
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (TikaException e) {
e.printStackTrace();
} finally {
LuceneContext.getInstance().releaseSearcher(searcher);
}
return pages;
}
private void indexMessage(Message msg) {
List<String> contents = new ArrayList<String>();
List<String> ans = new ArrayList<String>();
contents.add(msg.getContent());
List<Attachment> atts = attachmentDao.listByMessage(msg.getId());
for(Attachment att:atts) {
IndexUtil.attach2Index(contents, ans,att);
}
IndexField field = IndexUtil.msg2IndexField(msg);
field.setContent(contents);
field.setAtths(ans);
addIndex(field,false);
}
private void indexMessages(List<Message> messages) {
for(Message msg:messages) {
indexMessage(msg);
}
}
@Override
public void updateReconstructorIndex() {
/**
* 将数据库中的所有对象取出，创建相应的IndexField完成索引的重构
*/
try {
LuceneContext.getInstance().getNRTManager().deleteAll();
List<Message> messages = messageDao.list("from Message");
indexMessages(messages);
LuceneContext.getInstance().commitIndex();
tempIndexDao.delAll();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void updateSetIndex() {
List<TempIndex> tis = tempIndexDao.list("from TempIndex");
for(TempIndex ti:tis) {
if(ti.getType().equals(IndexUtil.MSG_TYPE)) {
Message msg = messageDao.load(ti.getObjId());
indexMessage(msg);
}
}
LuceneContext.getInstance().commitIndex();
tempIndexDao.delAll();
}
@Override
public void updateCommitIndex() {
tempIndexDao.delAll();
LuceneContext.getInstance().commitIndex();
}
}
public class CleanListener implements ServletContextListener {
private Timer indexTimer;
private WebApplicationContext wac = null;
private class IndexCommit extends TimerTask {
@Override
public void run() {
SystemContext.setRealPath(realPath);
System.out.println("索引进行了提交"+new Date());
IIndexService indexService = (IIndexService)wac.getBean("indexService");
indexService.updateCommitIndex();
}
}
@Override
public void contextInitialized(ServletContextEvent sce) {
//可以获取spring中BeanFactory,这个BeanFactory是在系统启动的时候就完成存储了
wac = WebApplicationContextUtils.getWebApplicationContext(sce.getServletContext());
System.out.println("------------------清理的启动程序已经开启（已经获取了"+wac+"）---------------------");
realPath = sce.getServletContext().getRealPath("");
timer = new Timer();
timer.scheduleAtFixedRate(new ClearDataTask(),50000, 300000);
indexTimer = new Timer();
indexTimer.scheduleAtFixedRate(new IndexCommit(), 600000, 600000);
}