solr使用中文，庖丁分词
1、分析器改造 net.paoding.analysis.analyzer.PaodingTokenizer.java
extends Tokenizer 原来是 extends TokenStream
2、net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory.java
package net.paoding.analysis.analyzer.solr;
import java.io.Reader;
import java.util.Map;
import net.paoding.analysis.analyzer.PaodingTokenizer;
import net.paoding.analysis.analyzer.TokenCollector;
import net.paoding.analysis.analyzer.impl.MaxWordLengthTokenCollector;
import net.paoding.analysis.analyzer.impl.MostWordsTokenCollector;
import net.paoding.analysis.knife.PaodingMaker;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.solr.analysis.BaseTokenizerFactory;
/**
* Created by IntelliJ IDEA. User: ronghao Date: 2007-11-3 Time: 14:40:59 中文切词
* 对庖丁切词的封装
*/
public class ChineseTokenizerFactory extends BaseTokenizerFactory {
/**
* 最多切分 默认模式
*/
public static final String MOST_WORDS_MODE = "most-words";
/**
* 按最大切分
*/
public static final String MAX_WORD_LENGTH_MODE = "max-word-length";
private String mode = null;
public void setMode(String mode) {
if (mode == null || MOST_WORDS_MODE.equalsIgnoreCase(mode)
|| "default".equalsIgnoreCase(mode)) {
this.mode = MOST_WORDS_MODE;
} else if (MAX_WORD_LENGTH_MODE.equalsIgnoreCase(mode)) {
this.mode = MAX_WORD_LENGTH_MODE;
} else {
throw new IllegalArgumentException(
"不合法的分析器Mode 参数设置:"
+ mode);
}
}
@Override
public void init(Map<String,String> args) {
super.init(args);
setMode(args.get("mode"));
}
public Tokenizer create(Reader input) {
return new PaodingTokenizer(input, PaodingMaker.make(),
createTokenCollector());
}
private TokenCollector createTokenCollector() {
if (MOST_WORDS_MODE.equals(mode))
return new MostWordsTokenCollector();
if (MAX_WORD_LENGTH_MODE.equals(mode))
return new MaxWordLengthTokenCollector();
throw new Error("never happened");
}
}
3、下载庖丁 将paoding-analysis.jar改造成paoding-analysis-solr.jar（将1,2覆盖进去这个包）
4、定义庖丁词典 paoding-dic-home.properties
paoding.dic.home=D:/WORK/DOCSET/solr/paoding-analysis-2.0.4-beta/dic/
5、替换原有的分词类 D:\WORK\DOCSET\solr\j-solr1\solr\conf\schema.xml
<fieldtype name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory" mode="most-words"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory" mode="most-words"/>
<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldtype>