(精华)2020年8月18日 C#基础知识点 搜索引擎Lucene的使用
lucene的基本说明
/// lucene.net：全文检索的工具包，不是应用，只是个类库，完成了全文检索的功能
/// 就是把数据拆分—存起来—查询时—拆分—匹配—结果
///
/// Analysis–分词器，负责把字符串拆分成原子，包含了标准分词，直接空格拆分
/// 项目中用的是盘古中文分词，
/// Document–数据结构，定义存储数据的格式
/// Index–索引的读写类
/// QueryParser–查询解析器，负责解析查询语句
/// Search—负责各种查询类，命令解析后得到就是查询类
/// Store—索引存储类，负责文件夹等等
/// Util—常见工具类库
///
/// lucene是全文搜索必备的，是大型系统必备的
///
/// Search：
/// TermQuery–单元查询 new Term(“title”,“张三”) title:张三
/// BoolenQuery—new Term(“title”,“张三”) and new Term(“title”,“李四”) title:张三 + title:李四
/// new Term(“title”,“张三”) or new Term(“title”,“李四”) title:张三 title:李四
/// WildcardQuery—通配符 new Term(“title”,“张?”) title:张？
/// new Term(“title”,“张*”) title:张*
/// PrefixQuery—前缀查询 以xx开头 title:张*
/// PhraseQuery—间隔距离 包含没有 包含提莫 而且二者距离不能超过5
/// title: “没有 提莫”~5
/// 没有蘑菇的提莫 没有蘑菇的蘑菇的蘑菇的提莫
/// FuzzyQuery—近似查询，ibhone----iphone title:ibhone~
/// RangeQuery—范围查询 [1,100] {1,100}
///
/// Lucene.Net一进一出，建立索引需要获取数据源，分词-保存到硬盘
/// 索引查找，
/// 自然会有些延迟，以前淘宝上架宝贝，第二天才能搜索的
/// 索引更新策略：1 数据跟新—丢一个队列—一个processor通过队列完成更新
/// 2 每一周全部索引一遍
///
/// lucene索引存的是原子–docid1，docid2，docid3
/// 不store可以大量节约空间；查找时原子匹配多个id;
第一步：lucene初始化索引
/// 基础的查询
FSDirectory dir = FSDirectory.Open(StaticConstant.TestIndexPath);
IndexSearcher searcher = new IndexSearcher(dir);//查找器
TermQuery query = new TermQuery(new Term("title", "图书馆"));//包含
TopDocs docs = searcher.Search(query, null, 10000);//找到的数据
foreach (ScoreDoc sd in docs.ScoreDocs)
{
Document doc = searcher.Doc(sd.Doc);
Console.WriteLine("***************************************");
Console.WriteLine(string.Format("id={0}", doc.Get("id")));
Console.WriteLine(string.Format("title={0}", doc.Get("title")));
Console.WriteLine(string.Format("time={0}", doc.Get("time")));
Console.WriteLine(string.Format("price={0}", doc.Get("price")));
Console.WriteLine(string.Format("content={0}", doc.Get("content")));
}
Console.WriteLine("1一共命中了{0}个", docs.TotalHits);
关键字查询
FSDirectory dir = FSDirectory.Open(StaticConstant.TestIndexPath);
IndexSearcher searcher = new IndexSearcher(dir);//查找器
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "title", new PanGuAnalyzer());//解析器
string keyword = "高中政治 人 教 新课 标 选修 生活 中的 法律常识";
Query query = parser.Parse(keyword);
TopDocs docs = searcher.Search(query, null, 10000);//找到的数据
int i = 0;
foreach (ScoreDoc sd in docs.ScoreDocs)
{
if (i++ < 1000)
{
Document doc = searcher.Doc(sd.Doc);
Console.WriteLine("***************************************");
Console.WriteLine(string.Format("id={0}", doc.Get("id")));
Console.WriteLine(string.Format("title={0}", doc.Get("title")));
Console.WriteLine(string.Format("time={0}", doc.Get("time")));
Console.WriteLine(string.Format("price={0}", doc.Get("price")));
}
}
Console.WriteLine($"一共命中{docs.TotalHits}");
多条件查询，除了关键字，时间，排序
FSDirectory dir = FSDirectory.Open(StaticConstant.TestIndexPath);
IndexSearcher searcher = new IndexSearcher(dir);//查找器
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "title", new PanGuAnalyzer());//解析器
string keyword = "高中政治 人 教 新课 标 选修 生活 中的 法律常识";
Query query = parser.Parse(keyword);
NumericRangeFilter<int> timeFilter = NumericRangeFilter.NewIntRange("time", 20190101, 20191231, true, true);//过滤
SortField sortPrice = new SortField("price", SortField.DOUBLE, false);//降序
SortField sortTime = new SortField("time", SortField.INT, true);//升序
Sort sort = new Sort(sortTime, sortPrice);//排序 哪个前哪个后
TopDocs docs = searcher.Search(query, timeFilter, 10000, sort);//找到的数据
int i = 0;
foreach (ScoreDoc sd in docs.ScoreDocs)
{
if (i++ < 1000)
{
Document doc = searcher.Doc(sd.Doc);
Console.WriteLine("***************************************");
Console.WriteLine(string.Format("id={0}", doc.Get("id")));
Console.WriteLine(string.Format("title={0}", doc.Get("title")));
Console.WriteLine(string.Format("time={0}", doc.Get("time")));
Console.WriteLine(string.Format("price={0}", doc.Get("price")));
}
}
Console.WriteLine("3一共命中了{0}个", docs.TotalHits);
/// 1 索引增删改查和分词处理
/// 2 京东数据多线程建立索引
/// 3 索引查询接口封装
///
/// Lucene–封装的lucene相关操作封装
///
/// LuceneAnalyze–负责完成查询关键字解析，尽可能拆分成原子数组
/// 如果只有一个词，prefix查询 苹果*
/// 如果是多个词，换成或者关系，
/// 都是为了更多的命中结果(贪婪搜索)
/// 做个关键词清理
///
/// LuceneBulid— BuildIndex–MergeIndex 多线程写不同子路径，完成后合并
/// 增加/删除索引 更新索引-只能先删除再更新
///
/// LuceneQuery—QueryIndexPage 支持关键字，支持范围过滤 支持排序
///
/// Processor—Lucene多线程建立索引
/// IndexBuilder 入口，启动多线程创建+完成后的Merge
/// IndexBuilderPerThread 每个线程是如何完成索引建立的
///
/// DataService–CommodityLucene对外提供的搜索封装
/// CommodityRepository-SqlHelper，完成数据库数据查询
批量索引建立
IndexBuilder.Build();
int total = 0;
string pricefilter = "[50,2000]";
string priceorderby = "price desc";
List<Commodity> commoditylist = CommodityLucene.QueryCommodity(1, 30, out total, "书", null, pricefilter, priceorderby);
foreach (Commodity commodity in commoditylist)
{
Console.WriteLine("title={0},price={1}", commodity.Title, commodity.Price);
}
/// public class IndexBuilderPerThread
{
private Logger logger = new Logger(typeof(IndexBuilderPerThread));
private int CurrentThreadNum = 0;
private string PathSuffix = "";
private CancellationTokenSource CTS = null;
public IndexBuilderPerThread(int threadNum, string pathSuffix, CancellationTokenSource cts)
{
this.CurrentThreadNum = threadNum;
this.PathSuffix = pathSuffix;
this.CTS = cts;
}
public void Process()
{
try
{
logger.Debug(string.Format("ThreadNum={0}开始创建", CurrentThreadNum));
CommodityRepository commodityRepository = new CommodityRepository();
ILuceneBulid builder = new LuceneBulid();
bool isFirst = true;
int pageIndex = 1;
while (!CTS.IsCancellationRequested)
{
List<CourseEntity> commodityList = commodityRepository.QueryList(CurrentThreadNum, pageIndex, 1000);
if (commodityList == null || commodityList.Count == 0)
{
break;
}
//else if (pageIndex == 11)
//{
// break;//为了测试 只做10000条数据
//}
else
{
builder.BuildIndex(commodityList, PathSuffix, isFirst);
logger.Debug(string.Format("ThreadNum={0}完成{1}条的创建", CurrentThreadNum, 1000 * pageIndex++));
isFirst = false;
}
}
}
catch (Exception ex)
{
CTS.Cancel();
logger.Error(string.Format("ThreadNum={0}出现异常", CurrentThreadNum), ex);
}
finally
{
logger.Debug(string.Format("ThreadNum={0}完成创建", CurrentThreadNum));
}
}
}
备注：相关类
/// public interface ILuceneAnalyze
{
/// public class LuceneAnalyze : ILuceneAnalyze
{
private Logger logger = new Logger(typeof(LuceneAnalyze));
#region AnalyzerKey
/// public class CommodityLucene
{
private static Logger logger = new Logger(typeof(CommodityLucene));
#region QueryCommodity
/// /// public class SqlHelper
{
private static Logger logger = new Logger(typeof(SqlHelper));
private static string ConnStr = ConfigurationManager.ConnectionStrings["TencentConn"].ConnectionString;
public static void ExecuteNonQuery(string sql)
{
//try
//{
using (SqlConnection sqlConn = new SqlConnection(ConnStr))
{
sqlConn.Open();
SqlCommand cmd = new SqlCommand(sql, sqlConn);
cmd.ExecuteNonQuery();//.ExecuteNonQueryAsync();//
}
//}
//catch (Exception ex)
//{
//}
//finally
//{
//}
}
public static List<T> QueryList<T>(string sql) where T : new()
{
using (SqlConnection sqlConn = new SqlConnection(ConnStr))
{
sqlConn.Open();
SqlCommand cmd = new SqlCommand(sql, sqlConn);
cmd.CommandTimeout = 120;
return TransList<T>(cmd.ExecuteReader());
}
}
public static void Insert<T>(T model, string tableName) where T : new()
{
string sql = GetInsertSql<T>(model, tableName);
ExecuteNonQuery(sql);
}
public static void InsertList<T>(List<T> list, string tableName) where T : new()
{
string sql = string.Join(" ", list.Select(t => GetInsertSql<T>(t, tableName)));
ExecuteNonQuery(sql);
}
#region Private
private static string GetInsertSql<T>(T model, string tableName)
{
StringBuilder sbSql = new StringBuilder();
StringBuilder sbFields = new StringBuilder();
StringBuilder sbValues = new StringBuilder();
Type type = model.GetType();
var properties = type.GetProperties();
foreach (PropertyInfo p in properties)
{
string name = p.Name;
if (!name.Equals("id", StringComparison.OrdinalIgnoreCase))
{
sbFields.AppendFormat("[{0}],", name);
sbValues.AppendFormat("'{0}',", p.GetValue(model));
}
}
sbSql.AppendFormat("INSERT INTO {0} ({1}) VALUES ({2});",tableName, sbFields.ToString().TrimEnd(','), sbValues.ToString().TrimEnd(','));
return sbSql.ToString();
}
private static List<T> TransList<T>(SqlDataReader reader) where T : new()
{
List<T> tList = new List<T>();
Type type = typeof(T);
var properties = type.GetProperties();
if (reader.Read())
{
do
{
T t = new T();
foreach (PropertyInfo p in properties)
{
p.SetValue(t, Convert.ChangeType(reader[p.Name], p.PropertyType));
}
tList.Add(t);
}
while (reader.Read());
}
return tList;
}
private static T TransModel<T>(SqlDataReader reader) where T : new()
{
T t = new T();
if (reader.Read())
{
do
{
Type type = typeof(T);
var properties = type.GetProperties();
foreach (PropertyInfo p in properties)
{
p.SetValue(t, Convert.ChangeType(reader[p.Name], p.PropertyType));
}
}
while (reader.Read());
}
return t;
}
#endregion Private
}