随便搞搞搜索引擎技术,刚开始分词而已,综合考察了几个比较有名的分词方法,决定还是用中科院的分词程序。其中C#有个开源的,且网上已经有用SharpICTCLAS为lucene.net写的分词接口了,不过想试试用好一点的分词程序的效果,所以选了2009共享版的。本人编程技术还是非常菜的,如有不对请大家指出。
分词接口的代码我是综合这两篇博客:http://ythzjk.javaeye.com/blog/334194,http://www.cnblogs.com/birdshover/archive/2009/04/03/1122305.html#1494633。
代码都不长也不难,仔细阅读后很容易根据把自己的分词程序放进lucene。据我观察似乎只要修改Tokenizer接口中next()方法就够了。不过他们的代码我觉得有个问题,每次初始化分词程序都是在Tokenizer的构造方法中,这就影响了分词效率,每分一句话(标题、正文)就要加载一次字典。于是我把加载词典的部分写成静态的。
调用ICTCLAS30.dll的方法,其实只用到了几个方法而已
using System;ICTCLASAnalyzer:
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using System.Collections;
namespace TestLucene
{
public class ICTCLASAnalyzer : Analyzer
{
//定义要过滤的词
public string[] CHINESE_ENGLISH_STOP_WORDS;
public string StopPath = @"E:/MyCsProj/TestLucene/Stopwords.txt";
public ICTCLASAnalyzer()
{
ArrayList StopWords= new ArrayList();
StreamReader reader = new StreamReader(StopPath, System.Text.Encoding.UTF8);
string noise = reader.ReadLine();
int i = 0;
while (!string.IsNullOrEmpty(noise))
{
StopWords.Add(noise);
noise = reader.ReadLine();
i++;
}
CHINESE_ENGLISH_STOP_WORDS = new String[i];
while (i>0)
{
i--;
CHINESE_ENGLISH_STOP_WORDS[i] = (string)StopWords[i];
}
StopWords.Clear();
}
/**//// Constructs a {@link StandardTokenizer} filtered by a {@link
/// StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
///
public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
{
TokenStream result = new ICTCLASTokenizer(reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
result = new StopFilter(result, CHINESE_ENGLISH_STOP_WORDS);
return result;
}
}
Tokenizer:
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using Lucene.Net.Analysis;
using System.IO;
namespace TestLucene
{
class ICTCLASTokenizer : Tokenizer
{
//result_t[] result; //本来我想直接使用这个结构的结果 但发现中英文识别字符长度不一样,分词之后原句子对应分词的位置不到了
int startIndex = 0;
int endIndex = 0;
int i = 0;
string[] pResult;
/**/
///
/// 待分词的句子
///
private string sentence;
/**/
/// Constructs a tokenizer for this Reader.
///
public static bool flag = ICTCLAS.ICTCLAS_Init(@"E:/MyCsProj/ttt/bin/Debug");
public static int userdic = ICTCLAS.ICTCLAS_ImportUserDict("userdict.txt");
public ICTCLASTokenizer(System.IO.TextReader reader)
{
this.input = reader;
sentence = input.ReadToEnd();
if(!flag)
{
System.Console.WriteLine("Init ICTCLAS failed!");
return;
}
string sResult= ICTCLAS.ICTCLAS_ParagraphProcess(sentence,1);
pResult = Regex.Split(sResult,@"(?<=/w) ");//直接获取分词结果字符串,在结果上分出词。
//Console.WriteLine(sResult);
}
/**/
/// 进行切词,返回数据流中下一个token或者数据流为空时返回null
///
public override Token Next()
{
Token token = null;
while (i < pResult.Length-1)
{
string word = pResult[i].Split('/')[0];
MatchCollection rw = Regex.Matches(word, @"/s");
int space = rw.Count;
startIndex=sentence.IndexOf(word,endIndex);
endIndex =startIndex+word.Length;
token = new Token(sentence.Substring(startIndex+space, word.Length-space),startIndex+space,endIndex);
i++;
// Console.WriteLine("word: {0},({1},{2})", sentence.Substring(startIndex + space, word.Length - space), startIndex + space, endIndex);
return token;
}
return null;
}
}
}