移动易实现基于Lucene的全文搜索
1、简单概述
全文搜索是指计算机索引程序通过扫描文章中的每一个词，对每一个词建立索引，当用户查询时，检索程序根据事先建立的索引进行查找并将结果返回给用户。本文主要介绍移动易系统基于Lucene的全文搜索(针对新闻部分)的实现。
2、前期准备
移动易项目(已托管在码云，自行下载导入，步骤见：移动易后台外部数据库连接的实现)
Maven使用经验
Spring boot 使用经验
MySQL 5.6（64位），具体安装及配置步骤见MySQL官网教程
FireFox(火狐)浏览器并安装Firebug插件
Lucene使用经验
3、具体实现步骤
3.1 添加maven依赖或者直接导入Lucene相关包(在项目Lucene文件夹下)
由于Elasticsearch底层基于Lucene并对其进行了扩展，所以只需在Maven中添加对Elasticsearch的依赖即可，打开pom.xml文件，添加如下依赖:
org.elasticsearch
elasticsearch
2.4.5
3.2 代码实现
3.2.1 创建LuceneUtils 类，实现Lucene的初始化操作，其中封装了分词器Analyzer、IndexWriter、IndexReader以及IndexSearcher等变量，并为之添加了Get和Set方法，然后将此类交给Spring容器管理。具体实现如下：
package com.sectong.lucene;
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.rest.core.Path;
import org.springframework.stereotype.Component;
@Component
public class LuceneUtils {
public static final Logger LOGGER = LoggerFactory.getLogger(LuceneUtils.class);
public String dir = "D:/Lucene/Index";
public Directory directory;
public IndexWriter iwriter;
public IndexWriterConfig iconfig;
public IndexReader ireader;
public IndexSearcher isearcher;
public Analyzer analyzer;
public LuceneUtils() throws IOException
{
analyzer = new StandardAnalyzer(Version.LUCENE_4_9);
iconfig = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
directory = FSDirectory.open(new File("D:/Lucene/Index"));
iwriter = new IndexWriter(directory, iconfig);
ireader = DirectoryReader.open(directory);
isearcher = new IndexSearcher(ireader);
LOGGER.info("LuceneUtils INITIALIZED");
}
/**
* 提交操作 实时更新
*
* @throws IOException
*/
public void commit() throws IOException
{
IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader)getIreader(), getIwriter(), false);
if(null!=newReader)
{
ireader.close();
ireader = newReader;
isearcher = new IndexSearcher(ireader);
}
}
/**
* 关闭索引
*
* @throws IOException
*/
public void close() throws IOException
{
directory.close();
ireader.close();
iwriter.close();
}
public String getDir() {
return dir;
}
public void setDir(String dir) {
this.dir = dir;
}
public Directory getDirectory() {
return directory;
}
public void setDirectory(Directory directory) {
this.directory = directory;
}
public IndexWriter getIwriter() {
return iwriter;
}
public void setIwriter(IndexWriter iwriter) {
this.iwriter = iwriter;
}
public IndexWriterConfig getIconfig() {
return iconfig;
}
public void setIconfig(IndexWriterConfig iconfig) {
this.iconfig = iconfig;
}
public IndexReader getIreader() {
return ireader;
}
public void setIreader(DirectoryReader ireader) {
this.ireader = ireader;
}
public IndexSearcher getIsearcher() {
return isearcher;
}
public void setIsearcher(IndexSearcher isearcher) {
this.isearcher = isearcher;
}
public Analyzer getAnalyzer() {
return analyzer;
}
public void setAnalyzer(Analyzer analyzer) {
this.analyzer = analyzer;
}
}
3.2.2 在Service包下创建搜索服务的接口NewsSearcherService以及其实现类NewsSearcherServiceImpl,
在NewsSearcherService接口中定义了索引的增删改查操作并在NewsSearcherServiceImpl类中实现了这些操作。
NewsSearcherService接口:
package com.sectong.service;
import java.io.IOException;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.TopDocs;
import com.sectong.domain.News;
public interface NewsSearchService {
Boolean createIndex(News news) throws IOException;
Boolean deleteIndex(News news) throws IOException;
Boolean updateIndex(News onews,News nnews) throws IOException;
List search(String[] fields,String message) throws ParseException, IOException;
void close() throws IOException;
}
NewsSearcherServiec 类:
package com.sectong.service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Service;
import com.sectong.controller.UserController;
import com.sectong.domain.News;
import com.sectong.lucene.LuceneUtils;
@Service
public class NewsSearchServiceImpl implements NewsSearchService {
private LuceneUtils lus;
private static final Logger LOGGER = LoggerFactory.getLogger(UserController.class);
@Autowired
public NewsSearchServiceImpl(LuceneUtils l)
{
lus = l;
LOGGER.info("NewsSearchService 初始化");
}
/**
* 创建索引
*
* @param News
*/
@Override
public Boolean createIndex(News news) throws IOException {
// TODO Auto-generated method stub
Document doc = new Document();
doc.add(new Field("title",news.getTitle(),TextField.TYPE_STORED));
doc.add(new Field("img",news.getImg(),TextField.TYPE_STORED));
doc.add(new Field("content",news.getContent(),TextField.TYPE_STORED));
doc.add(new Field("date",news.getDatetime().toString(),TextField.TYPE_STORED));
doc.add(new Field("user",news.getUser().getId().toString(),TextField.TYPE_STORED));
//doc.add(new Field("id",news.getId().toString(),TextField.TYPE_STORED));
lus.iwriter.addDocument(doc);
lus.commit();
LOGGER.info("成功添加新闻索引");
return true;
}
/**
* 删除索引
*
* @param News
*/
@Override
public Boolean deleteIndex(News news) throws IOException {
// TODO Auto-generated method stub
//MultiPhraseQuery query = new MultiPhraseQuery();
Term[] terms={new Term("title",news.getTitle()),new Term("date",news.getDatetime().toString()),new Term("user",news.getUser().getId().toString()),new Term("content",news.getContent())};
//query.add(terms);
lus.iwriter.deleteDocuments(terms);
LOGGER.info("成功删除新闻索引");
lus.commit();
return true;
}
/**
* 更新索引
*/
@Override
public Boolean updateIndex(News oldnews,News news) throws IOException {
// TODO Auto-generated method stub
Document doc = new Document();
doc.add(new Field("title",news.getTitle(),TextField.TYPE_STORED));
doc.add(new Field("img",news.getImg(),TextField.TYPE_STORED));
doc.add(new Field("content",news.getContent(),TextField.TYPE_STORED));
doc.add(new Field("date",news.getDatetime().toString(),TextField.TYPE_STORED));
doc.add(new Field("user",news.getUser().getId().toString(),TextField.TYPE_STORED));
lus.iwriter.updateDocument(new Term("title",oldnews.getTitle()), doc);
lus.commit();
LOGGER.info("成功更新索引");
return null;
}
/**
* 搜索操作
*
* @param fields
* @param num
* @param message
*/
@Override
public List search(String[] fields,String message) throws ParseException, IOException {
// TODO Auto-generated method stub
MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_4_9,fields, lus.analyzer);
Query query = parser.parse(message);
TopDocs topdocs = lus.getIsearcher().search(query,100);
ScoreDoc[] hits = topdocs.scoreDocs;
List result = new ArrayList();
if(hits.length!=0)
{
for(ScoreDoc s :hits)
{
Document d = lus.isearcher.doc(s.doc);
result.add(d);
}
}
return result;
}
/**
* 关闭索引
*
*/
@Override
public void close() throws IOException
{
lus.close();
}
}
3.2.3 通过Controller类实现逻辑处理，在controller包中创建SearchController类，为该类添加@Controller注解，并在该类中为搜索请求映射方法doSearch(),具体实现如下：
package com.sectong.controller;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import com.sectong.domain.News;
import com.sectong.repository.NewsRepository;
import com.sectong.service.NewsSearchService;
import com.sectong.service.NewsService;
@Controller
public class SearchController {
private NewsSearchService newssearchservice;
private NewsService newsservice;
private NewsRepository newsrepository;
@Autowired
public SearchController(NewsSearchService newssearchservice,NewsService newsService,NewsRepository newsRepository)
{
this.newssearchservice = newssearchservice;
this.newsservice = newsService;
this.newsrepository = newsRepository;
}
@PreAuthorize("hasRole('ROLE_ADMIN')")
@RequestMapping("/admin/news/search")
public String DoSearch(@RequestParam(value="message")String message,Model model) throws ParseException, IOException
{
System.out.println(message);
if(null==message||"".equals(message))
{
return "redirect:/admin/news";
}
else
{
long startime = System.currentTimeMillis();
String [] fields ={"title","content"};
List documents = newssearchservice.search(fields, 10, message);
long endtime = System.currentTimeMillis();
long dtime = endtime - startime;
System.out.println("共找到:"+documents.size()+"条记录,用时:"+dtime+"毫秒");
//newssearchservice.close();
List results = new ArrayList();
if(documents.size()!=0)
{
for(Document doc : documents)
{
String tvalue = doc.get("title");
String cvalue = doc.get("content");
List r = newsrepository.findByTitleAndContent(tvalue, cvalue);
results.addAll(r);
}
model.addAttribute("newslist", results);
}
String resultmessage = "共找到"+documents.size()+"条记录，用时"+dtime+"毫秒";
model.addAttribute("resultmessage", resultmessage);
return "admin/news";
}
}
}
3.2.4 在AdminController中实现对新闻增删改的同时创建删除和修改相应索引：
package com.sectong.controller;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import com.sectong.domain.News;
import com.sectong.domain.User;
import com.sectong.repository.NewsRepository;
import com.sectong.repository.UserRepository;
import com.sectong.service.NewsSearchService;
import com.sectong.service.UserService;
@Controller
public class AdminController {
private UserRepository userRepository;
private NewsRepository newsRepository;
private NewsSearchService newssearchService;
private UserService userService;
@Autowired
public AdminController(NewsRepository newsRepository, UserRepository userRepository, UserService userService,NewsSearchService newssearchService) {
this.userRepository = userRepository;
this.newsRepository = newsRepository;
this.userService = userService;
this.newssearchService = newssearchService;
}
/**
* 管理主界面
*
* @param model
* @return
*/
@GetMapping("/admin/")
public String adminIndex(Model model) {
model.addAttribute("dashboard", true);
model.addAttribute("userscount", userRepository.count());
model.addAttribute("newscount", newsRepository.count());
return "admin/index";
}
/**
* 用户管理
*
* @param model
* @return
*/
@GetMapping("/admin/user")
public String adminUser(Model model) {
model.addAttribute("user", true);
return "admin/user";
}
/**
* 新闻管理
*
* @param model
* @return
* @throws ParseException
*/
@GetMapping("/admin/news")
public String adminNews(Model model) throws ParseException {
model.addAttribute("news", true);
Iterable newslist = newsRepository.findAll();
model.addAttribute("newslist", newslist);
return "admin/news";
}
/**
* 新闻增加表单
*
* @param model
* @return
*/
@PreAuthorize("hasRole('ROLE_ADMIN')")
@GetMapping("/admin/news/add")
public String newsAdd(Model model) {
model.addAttribute("newsAdd", new News());
return "admin/newsAdd";
}
/**
* 新闻修改表单
*
* @param model
* @return
*/
@PreAuthorize("hasRole('ROLE_ADMIN')")
@GetMapping("/admin/news/edit")
public String newsEdit(Model model, @RequestParam Long id) {
model.addAttribute("newsEdit", newsRepository.findOne(id));
return "admin/newsEdit";
}
/**
* 新闻修改提交操作
*
* @param news
* @return
* @throws IOException
* @throws ParseException
*/
@PreAuthorize("hasRole('ROLE_ADMIN')")
@PostMapping("/admin/news/edit")
public String newsSubmit(@ModelAttribute News news) throws IOException, ParseException {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
Date d = dateFormat.parse(dateFormat.format(new Date()));
news.setDatetime(d);
User user = userService.getCurrentUser();
news.setUser(user);
if(null!=news.getId())
{
News oldnews = newsRepository.findOne(news.getId());
newssearchService.updateIndex(oldnews, news);
}
else
{
newssearchService.createIndex(news);
}
newsRepository.save(news);
return "redirect:/admin/news";
}
/**
* 新闻删除操作
*
* @param model
* @param id
* @return
* @throws IOException
*/
@PreAuthorize("hasRole('ROLE_ADMIN')")
@GetMapping("/admin/news/del")
public String delNews(Model model, @RequestParam Long id) throws IOException {
News news = newsRepository.findOne(id);
newssearchService.deleteIndex(news);
newsRepository.delete(id);
return "redirect:/admin/news";
}
}
4、实现效果
5、代码下载
具体内容在 移动易git代码库 mysql-connector 分支下：
http://git.oschina.net/sectong/yidongyi/tree/f9f8661e27ed8bd41a7249580381fa43482c2f6f