Lucene 的增量索引 索引库的自动更新
前言：最近写了个小程序，让索引库定时自动更新，可以让你新增的数据和修改的数据都能进索引库，唯一的一个遗憾是不能将删除掉的数据剔除出去，一直在思考中。废话也不多说，上代码给瞅瞅，然后也希望有更好的解决方法的人能给我留言发表意见。
1.
主要的方法增量索引的创建:
///
/// 邮件的增量索引
/// 定时器跑的方法
///
public void IncreaseEdmIndex() {
//判断有无字典
if (System.IO.Directory.Exists(edmIndexPath) == true)
{
FSDirectory directory = directory_edm;
IndexReader reader = DirectoryReader.Open(directory);
DateTime oldCreateIndexTime = Convert.ToDateTime(reader.Document(reader.NumDocs()-1).Get("indexData").ToString().Trim());
//数据库中的所有数据
List
List
List
foreach (OutapiChannelApply item in dataAll)
{
bool addf = IsInIndex(item.ApplyId.ToString().Trim(), reader);
if (addf == true)
{
addData.Add(item);
}
else {
updData.Add(item);
}
}
if (updData.Count > 0)
{
UpdDataListToIndex(updData, reader);
}
reader.Close();
IndexWriter writer = new IndexWriter(directory, analyzer, false, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);
if (addData.Count > 0 && updData.Count > 0)
{
AddDataListToIndex(updData, writer);
AddDataListToIndex(addData, writer);
}
if (addData.Count > 0 && updData.Count == 0)
{
AddDataListToIndex(addData, writer);
}
writer.Optimize();
writer.Close();
directory.Close();
}
else {
CreateEdmIndex();
}
}
2.关联的方法：
///
/// 功能描述：判断数据库中的某条数据是否已经索引了
///
/// 数据库记录的id字段值
/// IndexReader的对象
///
private bool IsInIndex(String id, IndexReader p_indexReader)
{
bool flag = true;
for (int i = 0; i < p_indexReader.NumDocs(); i++)
{
Document doc = p_indexReader.Document(i);
if (id.Equals(doc.Get("id")))
{
flag = false;
}
}
return flag;
}
///
/// 邮件设置里面的 添加数据进索引里面的文本转换
///
///
///
private Document ToDocumentByData(OutapiChannelApply resultData)
{
Document document = new Document();
document.Add(new Field("id", resultData.ApplyId.ToString().Trim(), Field.Store.YES, Field.Index.NOT_ANALYZED));//NOT_ANALYZED--不分词
document.Add(new Field("channelId", resultData.ChannelId.ToString().Trim(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("platForm", resultData.Platform.ToString().Trim(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("updateTime", resultData.UpdateTime.ToString().Trim(), Field.Store.YES, Field.Index.NOT_ANALYZED));
document.Add(new Field("name", resultData.ShopName.Trim().ToString(), Field.Store.YES, Field.Index.ANALYZED, Lucene.Net.Documents.Field.TermVector.WITH_POSITIONS_OFFSETS));
document.Add(new Field("indexData", System.DateTime.Now.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); //DateTools.DateToString(System.DateTime.Now,DateTools.Resolution.DAY)
return document;
}
3.创建索引：
///
/// 创建索引
/// edm_recevie_info和erp_channel
///
public void CreateEdmIndex() {
FSDirectory directory = directory_edm;
//思考判断 字典是否存在并有数据
//有的话做追加 没有就是添加所有的数据索引
bool isUpdate = IndexReader.IndexExists(directory);
if (isUpdate)
{
if (IndexWriter.IsLocked(directory))
{
IndexWriter.Unlock(directory);
}
}
//IndexWriter用于向索引库写内容
//IndexWriter的第三个参数的解释：true表示删除之前的重新写入 false:表示追加
//使用IndexWriter打开directory时会自动对索引库文件上锁 多人同时操作并发问题
IndexWriter writer = new IndexWriter(directory, analyzer, !isUpdate, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeFactor(100);
writer.SetMaxBufferedDocs(100);
writer.SetMaxMergeDocs(1000);
writer.SetUseCompoundFile(true);
List
foreach (OutapiChannelApply item in getEdmData)
{
Document document = ToDocumentByData(item);
writer.AddDocument(document);//将文档写入索引库
}
writer.Close();
directory.Close();//不要忘了Close，否则索引结果搜不到
}
//增量索引 批量添加索引库
public void AddDataListToIndex(List
{
if (resultList.Count > 0)
{
foreach (OutapiChannelApply item in resultList)
{
Document document = ToDocumentByData(item);
writer.AddDocument(document);//将文档写入索引库
}
}
}
//增量索引 第一步去索引库删除数据
public void UpdDataListToIndex(List
{
if (resultList.Count > 0)
{
foreach (OutapiChannelApply item in resultList)
{
Term term = new Term("id", item.ApplyId.ToString().Trim());
Query query = new TermQuery(term);
reader.DeleteDocuments(term);
}
}
}
4.弄定时器定时更行：
public class TimerToIndex
{
static string edmSwitch = new LuceneAreaProvider().GetSwitch();
static Timer updateIndex = new Timer(new TimerCallback(TimerToDo), null, Timeout.Infinite, Timeout.Infinite);
static ILog Log = LogManager.GetCurrentClassLogger();
static void TimerToDo(object obj)
{
Log.TraceFormat("{0}更新edm数据", DateTime.Now.ToString());
new LuceneHelper().IncreaseEdmIndex();
}
public static void Start(long a,long b) {
if (edmSwitch.Equals("yes"))
{
updateIndex.Change(a, b);
}
}
public static void Stop() {
updateIndex.Change(Timeout.Infinite, Timeout.Infinite); //Timeout.Infinite 无限长的时间
}
}
5.Global文件添加
protected void Application_Start(object sender, EventArgs e)
{
AutofacRegisterConfig.Resolver();
long a = 0; //开始时间 0表示立即开始
long b = 86400000; //间隔时间 单位是毫秒 一天
TimerToIndex.Start(a, b);
}