Elasticsearch ik分词器加载远程数据库中的热词库
1、下载 elasticsearch-analysis-ik 源码包
下载地址：
https://github.com/medcl/elasticsearch-analysis-ik/releases
2、修改源码
org.wltea.analyzer.dic.Dictionary 单例类的初始化方法 initial，在这里需要创建一个我们自定义的线程，并且启动它
/**
* 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化
* 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段
*
* @return Dictionary
*/
public static synchronized void initial(Configuration cfg) {
if (singleton == null) {
synchronized (Dictionary.class) {
if (singleton == null) {
singleton = new Dictionary(cfg);
singleton.loadMainDict();
singleton.loadSurnameDict();
singleton.loadQuantifierDict();
singleton.loadSuffixDict();
singleton.loadPrepDict();
singleton.loadStopWordDict();
//创建一个自定义的线程，并且启动它
new Thread(new HotDictReloadThread()).start();
new Thread(new StopDictReloadThread()).start();
if(cfg.isEnableRemoteDict()){
// 建立监控线程
for (String location : singleton.getRemoteExtDictionarys()) {
// 10 秒是初始延迟可以修改的 60是间隔时间 单位秒
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
for (String location : singleton.getRemoteExtStopWordDictionarys()) {
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
}
}
}
}
}
HotDictReloadThread类：不断调用Dictionary.getSingleton().reLoadMainDict()，去重新加载词典
/**
* 死循环，不断调用Dictionary.getSingleton().reLoadMainDict()，去重新加载词典
*/
private static class HotDictReloadThread implements Runnable {
@Override
public void run() {
while(true) {
logger.info("[==========]reload hot dict from mysql......");
Dictionary.getSingleton().reLoadMainDict();
}
}
}
private static class StopDictReloadThread implements Runnable {
@Override
public void run() {
while (true){
Dictionary.getSingleton().loadDBStopWordsDict();
}
}
}
Dictionary类：更新词典 reLoadMainDict()
void reLoadMainDict() {
logger.info("重新加载词典...");
// 新开一个实例加载词典，减少加载过程对当前词典使用的影响
Dictionary tmpDict = new Dictionary(configuration);
tmpDict.configuration = getSingleton().configuration;
tmpDict.loadMainDict();
tmpDict.loadStopWordDict();
tmpDict.loadDBMainDict();
tmpDict.loadDBStopWordsDict();
_MainDict = tmpDict._MainDict;
_StopWords = tmpDict._StopWords;
logger.info("重新加载词典完毕...");
}
从数据库加载词典
/**
* 从DB数据库加载热更新字典
*/
public void loadDBMainDict() {
ResultSet mainRs = null;
Connection conn = null;
Statement stmt = null;
try {
// logger.info("[==========]query hot dict from DB, " + myProps.getProperty("jdbc.reload.sql") + "......");
if (conn==null || stmt==null) {
Class.forName(myProps.getProperty("jdbc.driver"));
conn = DriverManager.getConnection(
myProps.getProperty("jdbc.url"),
myProps.getProperty("jdbc.user"),
myProps.getProperty("jdbc.password"));
}
String main_sql = myProps.getProperty("jdbc.reload.main.sql");
if (!StringUtils.isNullOrEmpty(main_sql)) {
stmt = conn.createStatement();
mainRs = stmt.executeQuery(main_sql);
ArrayList mainList = new ArrayList<>();
while (mainRs.next()) {
String theWord = mainRs.getString("words");
mainList.add(theWord);
_MainDict.fillSegment(theWord.trim().toCharArray());
}
if (mainList.size()!=mainWords){
logger.info("reload the personalized words from DB: "+mainList.toString());
mainWords=mainList.size();
}
mainList.clear();
Thread.sleep(Integer.valueOf(myProps.getProperty("jdbc.reload.interval")));
}
} catch (Exception e) {
logger.error("erorr", e);
} finally {
try {
if (mainRs != null) {
mainRs.close();
}
if (stmt != null){
stmt.close();
}
if (conn != null){
conn.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 从DB加载停用词
*/
private void loadDBStopWordsDict() {
Connection conn = null;
Statement stmt = null;
ResultSet stopRs = null;
try {
// logger.info("[==========]query hot stopword dict from DB, " + myProps.getProperty("jdbc.reload.stopword.sql") + "......");
if (conn==null || stmt==null) {
Class.forName(myProps.getProperty("jdbc.driver"));
conn = DriverManager.getConnection(
myProps.getProperty("jdbc.url"),
myProps.getProperty("jdbc.user"),
myProps.getProperty("jdbc.password"));
stmt = conn.createStatement();
}
String stop_sql = myProps.getProperty("jdbc.reload.stopWord.sql");
if (!StringUtils.isNullOrEmpty(stop_sql)) {
stopRs = stmt.executeQuery(stop_sql);
ArrayList stopList = new ArrayList<>();
while (stopRs.next()) {
String theWord = stopRs.getString("words");
stopList.add(theWord);
_StopWords.fillSegment(theWord.trim().toCharArray());
}
if (stopList.size()!=stopWords){
logger.info("reload the stop words from DB: "+stopList.toString());
stopWords=stopList.size();
}
stopList.clear();
Thread.sleep(Integer.valueOf(myProps.getProperty("jdbc.reload.interval")));
}
} catch (Exception e) {
logger.error("erorr", e);
}finally {
try {
if (stopRs != null) {
stopRs.close();
}
if (stmt != null){
stmt.close();
}
if (conn != null){
conn.close();
}
} catch (SQLException e) {
e.printStackTrace();
}
}
}
配置文件 config/jdbc-reload.properties
jdbc.url=jdbc:postgresql://192.168.***.***:5432/search
jdbc.user=***
jdbc.password=***
jdbc.reload.main.sql=SELECT words FROM public.sys_personal_t
jdbc.reload.stopWord.sql=SELECT words FROM public.sys_stopwords_t
jdbc.reload.interval=10000
jdbc.driver=org.postgresql.Driver
修改 src/main/assemblies/plugin.xml
//将 mysql 的 jar 包打包进 zip 包
true
true
mysql:mysql-connector-java
true
true
org.postgresql:postgresql
修改 pom.xml
7.2.0
3、错误解决
java.sql.SQLNonTransientConnectionException: Could not create connection to database server.
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:526) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:513) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:505) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:479) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:1779) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:1596) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.ConnectionImpl.(ConnectionImpl.java:633) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:347) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:219) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at java.sql.DriverManager.getConnection(DriverManager.java:664) ~[?:1.8.0_181]
at java.sql.DriverManager.getConnection(DriverManager.java:247) ~[?:1.8.0_181]
at org.wltea.analyzer.dic.Dictionary.loadMySQLExtDict(Dictionary.java:464) [elasticsearch-analysis-ik-7.2.0.jar:?]
at org.wltea.analyzer.dic.Dictionary.loadMainDict(Dictionary.java:402) [elasticsearch-analysis-ik-7.2.0.jar:?]
at org.wltea.analyzer.dic.Dictionary.reLoadMainDict(Dictionary.java:698) [elasticsearch-analysis-ik-7.2.0.jar:?]
at org.wltea.analyzer.dic.Dictionary$HotDictReloadThread.run(Dictionary.java:714) [elasticsearch-analysis-ik-7.2.0.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
Caused by: java.security.AccessControlException: access denied ("java.net.SocketPermission" "192.168.108.140:3306" "connect,resolve")
at java.security.AccessControlContext.checkPermission(AccessControlContext.java:472) ~[?:1.8.0_181]
at java.security.AccessController.checkPermission(AccessController.java:884) ~[?:1.8.0_181]
at java.lang.SecurityManager.checkPermission(SecurityManager.java:549) ~[?:1.8.0_181]
at java.lang.SecurityManager.checkConnect(SecurityManager.java:1051) ~[?:1.8.0_181]
at java.net.Socket.connect(Socket.java:584) ~[?:1.8.0_181]
at com.mysql.cj.core.io.StandardSocketFactory.connect(StandardSocketFactory.java:202) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.mysqla.io.MysqlaSocketConnection.connect(MysqlaSocketConnection.java:57) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.mysqla.MysqlaSession.connect(MysqlaSession.java:122) ~[mysql-connector-java-6.0.6.jar:6.0.6]
at com.mysql.cj.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:1726) ~[mysql-connector-java-6.0.6.jar:6.0.6]
修改 $JAVA_HOME//jre/lib/security/java.policy
grant {
// needed because of the hot reload functionality
permission java.net.SocketPermission "*", "connect,resolve";
};