移动易实现基于Lucene的全文搜索

移动易实现基于Lucene的全文搜索

1、简单概述

全文搜索是指计算机索引程序通过扫描文章中的每一个词,对每一个词建立索引,当用户查询时,检索程序根据事先建立的索引进行查找并将结果返回给用户。本文主要介绍移动易系统基于Lucene的全文搜索(针对新闻部分)的实现。

2、前期准备

  1. 移动易项目(已托管在码云,自行下载导入,步骤见:移动易后台外部数据库连接的实现)

  2. Maven使用经验

  3. Spring boot 使用经验

  4. MySQL 5.6(64位),具体安装及配置步骤见MySQL官网教程

  5. FireFox(火狐)浏览器并安装Firebug插件

  6. Lucene使用经验

3、具体实现步骤

3.1 添加maven依赖或者直接导入Lucene相关包(在项目Lucene文件夹下)

  1. 由于Elasticsearch底层基于Lucene并对其进行了扩展,所以只需在Maven中添加对Elasticsearch的依赖即可,打开pom.xml文件,添加如下依赖:

    
    
    
        org.elasticsearch
        elasticsearch
        2.4.5
    

3.2 代码实现

3.2.1 创建LuceneUtils 类,实现Lucene的初始化操作,其中封装了分词器Analyzer、IndexWriter、IndexReader以及IndexSearcher等变量,并为之添加了Get和Set方法,然后将此类交给Spring容器管理。具体实现如下:

    
    package com.sectong.lucene;
    
    import java.io.File;
    import java.io.IOException;
    
    import org.apache.lucene.analysis.Analyzer;
    import org.apache.lucene.analysis.standard.StandardAnalyzer;
    import org.apache.lucene.index.DirectoryReader;
    import org.apache.lucene.index.IndexReader;
    import org.apache.lucene.index.IndexWriter;
    import org.apache.lucene.index.IndexWriterConfig;
    import org.apache.lucene.search.IndexSearcher;
    import org.apache.lucene.store.Directory;
    import org.apache.lucene.store.FSDirectory;
    import org.apache.lucene.util.Version;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.data.rest.core.Path;
    import org.springframework.stereotype.Component;
    
    @Component
    public class LuceneUtils {
    
        
        public static final Logger LOGGER = LoggerFactory.getLogger(LuceneUtils.class);
        public String dir = "D:/Lucene/Index";
        public Directory directory;
        public IndexWriter iwriter;
        public IndexWriterConfig iconfig;  
        public IndexReader ireader;
        public IndexSearcher isearcher;
        public Analyzer analyzer;
        
        public LuceneUtils() throws IOException
        {
            analyzer = new StandardAnalyzer(Version.LUCENE_4_9);
            iconfig = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
            directory = FSDirectory.open(new File("D:/Lucene/Index"));
            iwriter = new IndexWriter(directory, iconfig);
            ireader = DirectoryReader.open(directory);
            isearcher = new IndexSearcher(ireader);
            LOGGER.info("LuceneUtils INITIALIZED");
        }
        /**
         * 提交操作 实时更新
         * 
         * @throws IOException
         */
        public void commit() throws IOException
        {
            IndexReader newReader = DirectoryReader.openIfChanged((DirectoryReader)getIreader(), getIwriter(), false);
            if(null!=newReader)
            {
                ireader.close();
                ireader = newReader;
                isearcher = new IndexSearcher(ireader);
            }
            
        }
        
        /**
         * 关闭索引
         * 
         * @throws IOException
         */
        
        public void close() throws IOException
        {
            directory.close();
            ireader.close();
            iwriter.close();
        }
        
        public String getDir() {
            return dir;
        }
    
        public void setDir(String dir) {
            this.dir = dir;
        }
    
        public Directory getDirectory() {
            return directory;
        }
    
        public void setDirectory(Directory directory) {
            this.directory = directory;
        }
    
        public IndexWriter getIwriter() {
            return iwriter;
        }
    
        public void setIwriter(IndexWriter iwriter) {
            this.iwriter = iwriter;
        }
    
        public IndexWriterConfig getIconfig() {
            return iconfig;
        }
    
        public void setIconfig(IndexWriterConfig iconfig) {
            this.iconfig = iconfig;
        }
    
        public IndexReader getIreader() {
            return ireader;
        }
    
        public void setIreader(DirectoryReader ireader) {
            this.ireader = ireader;
        }
    
        public IndexSearcher getIsearcher() {
            return isearcher;
        }
    
        public void setIsearcher(IndexSearcher isearcher) {
            this.isearcher = isearcher;
        }
    
    
        public Analyzer getAnalyzer() {
            return analyzer;
        }
    
    
        public void setAnalyzer(Analyzer analyzer) {
            this.analyzer = analyzer;
        }
        
    }

3.2.2 在Service包下创建搜索服务的接口NewsSearcherService以及其实现类NewsSearcherServiceImpl,
在NewsSearcherService接口中定义了索引的增删改查操作并在NewsSearcherServiceImpl类中实现了这些操作。
NewsSearcherService接口:


    package com.sectong.service;
    
    import java.io.IOException;
    import java.util.List;
    
    import org.apache.lucene.document.Document;
    import org.apache.lucene.queryparser.classic.ParseException;
    import org.apache.lucene.search.TopDocs;
    
    import com.sectong.domain.News;
    
    public interface NewsSearchService {
    
        Boolean createIndex(News news) throws IOException;
        Boolean deleteIndex(News news) throws IOException;
        Boolean updateIndex(News onews,News nnews) throws IOException;
        List search(String[] fields,String message) throws ParseException, IOException;
        void close() throws IOException;
        
    }

NewsSearcherServiec 类:

    package com.sectong.service;
    
    import java.io.IOException;
    import java.util.ArrayList;
    import java.util.List;
    
    import org.apache.lucene.document.Document;
    import org.apache.lucene.document.Field;
    import org.apache.lucene.document.TextField;
    import org.apache.lucene.index.Term;
    import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
    import org.apache.lucene.queryparser.classic.ParseException;
    import org.apache.lucene.search.MultiPhraseQuery;
    import org.apache.lucene.search.PhraseQuery;
    import org.apache.lucene.search.Query;
    import org.apache.lucene.search.ScoreDoc;
    import org.apache.lucene.search.TopDocs;
    import org.apache.lucene.util.Version;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.context.annotation.Bean;
    import org.springframework.stereotype.Service;
    
    import com.sectong.controller.UserController;
    import com.sectong.domain.News;
    import com.sectong.lucene.LuceneUtils;
    
    
    @Service
    public class NewsSearchServiceImpl implements NewsSearchService {
    
        private LuceneUtils lus;
        private static final Logger LOGGER = LoggerFactory.getLogger(UserController.class);
        @Autowired
        public NewsSearchServiceImpl(LuceneUtils l)
        {
            lus = l;
            LOGGER.info("NewsSearchService 初始化");
        }
        
        /**
         * 创建索引
         * 
         * @param News
         */
        @Override
        public Boolean createIndex(News news) throws IOException {
            // TODO Auto-generated method stub
            
            Document doc = new Document();
            doc.add(new Field("title",news.getTitle(),TextField.TYPE_STORED));
            doc.add(new Field("img",news.getImg(),TextField.TYPE_STORED));
            doc.add(new Field("content",news.getContent(),TextField.TYPE_STORED));
            doc.add(new Field("date",news.getDatetime().toString(),TextField.TYPE_STORED));
            doc.add(new Field("user",news.getUser().getId().toString(),TextField.TYPE_STORED));
            //doc.add(new Field("id",news.getId().toString(),TextField.TYPE_STORED));
            lus.iwriter.addDocument(doc);
            lus.commit();
            LOGGER.info("成功添加新闻索引");
            return true;
        }
    
        /**
         * 删除索引
         * 
         * @param News
         */
        @Override
        public Boolean deleteIndex(News news) throws IOException {
            // TODO Auto-generated method stub
            //MultiPhraseQuery query = new MultiPhraseQuery();
            Term[] terms={new Term("title",news.getTitle()),new Term("date",news.getDatetime().toString()),new Term("user",news.getUser().getId().toString()),new Term("content",news.getContent())};
            //query.add(terms);
            lus.iwriter.deleteDocuments(terms);
            LOGGER.info("成功删除新闻索引");
            lus.commit();
            return true;
        }
        
        
        /**
         * 更新索引
         */
        @Override
        public Boolean updateIndex(News oldnews,News news) throws IOException {
            // TODO Auto-generated method stub
            Document doc = new Document();
            doc.add(new Field("title",news.getTitle(),TextField.TYPE_STORED));
            doc.add(new Field("img",news.getImg(),TextField.TYPE_STORED));
            doc.add(new Field("content",news.getContent(),TextField.TYPE_STORED));
            doc.add(new Field("date",news.getDatetime().toString(),TextField.TYPE_STORED));
            doc.add(new Field("user",news.getUser().getId().toString(),TextField.TYPE_STORED));
            
            lus.iwriter.updateDocument(new Term("title",oldnews.getTitle()), doc);
            lus.commit();
            LOGGER.info("成功更新索引");
            
            return null;
        }
    
        /**
         * 搜索操作
         * 
         * @param fields
         * @param num 
         * @param message
         */
        @Override
        public List search(String[] fields,String message) throws ParseException, IOException {
            // TODO Auto-generated method stub
            MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_4_9,fields, lus.analyzer);
            Query query = parser.parse(message);
            TopDocs topdocs = lus.getIsearcher().search(query,100);
            ScoreDoc[] hits = topdocs.scoreDocs;
            List result = new ArrayList();
            if(hits.length!=0)
            {
                for(ScoreDoc s :hits)
                {
                    Document d = lus.isearcher.doc(s.doc);
                    result.add(d);
                    
                }
            }
            
            return result;
        }
    
        /**
         * 关闭索引
         * 
         */
        @Override
        public void close() throws IOException
        {
            lus.close();
        }
    
    }

3.2.3 通过Controller类实现逻辑处理,在controller包中创建SearchController类,为该类添加@Controller注解,并在该类中为搜索请求映射方法doSearch(),具体实现如下:

    package com.sectong.controller;
    
    import java.io.IOException;
    import java.util.ArrayList;
    import java.util.List;
    
    import org.apache.lucene.document.Document;
    import org.apache.lucene.queryparser.classic.ParseException;
    import org.apache.lucene.search.ScoreDoc;
    import org.apache.lucene.search.TopDocs;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.security.access.prepost.PreAuthorize;
    import org.springframework.stereotype.Controller;
    import org.springframework.ui.Model;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RequestParam;
    
    import com.sectong.domain.News;
    import com.sectong.repository.NewsRepository;
    import com.sectong.service.NewsSearchService;
    import com.sectong.service.NewsService;
    
    
    @Controller
    public class SearchController {
    
        
        private NewsSearchService newssearchservice;
        private NewsService newsservice;
        private NewsRepository newsrepository;
        
        @Autowired
        public SearchController(NewsSearchService newssearchservice,NewsService newsService,NewsRepository newsRepository)
        {
            this.newssearchservice = newssearchservice;
            this.newsservice = newsService;
            this.newsrepository = newsRepository;
        }
        
        @PreAuthorize("hasRole('ROLE_ADMIN')")
        @RequestMapping("/admin/news/search")
        public String DoSearch(@RequestParam(value="message")String message,Model model) throws ParseException, IOException
        {
            System.out.println(message);
            if(null==message||"".equals(message))
            {
                return "redirect:/admin/news";
            }
            else
            {
            long startime = System.currentTimeMillis();
            String [] fields ={"title","content"};
            List documents = newssearchservice.search(fields, 10, message);
            long endtime = System.currentTimeMillis();
            long dtime = endtime - startime;
            System.out.println("共找到:"+documents.size()+"条记录,用时:"+dtime+"毫秒");
            //newssearchservice.close();
            List results = new ArrayList();
            if(documents.size()!=0)
            {
                for(Document doc : documents)
                {
                    String tvalue = doc.get("title");
                    String cvalue = doc.get("content");
                    List r = newsrepository.findByTitleAndContent(tvalue, cvalue);
                    results.addAll(r);
                }
                model.addAttribute("newslist", results);
                
            }
            String resultmessage = "共找到"+documents.size()+"条记录,用时"+dtime+"毫秒";
            model.addAttribute("resultmessage", resultmessage);
            return "admin/news";
            }
        }
        
    }

3.2.4 在AdminController中实现对新闻增删改的同时创建删除和修改相应索引:

    package com.sectong.controller;
    
    import java.io.IOException;
    import java.text.ParseException;
    import java.text.SimpleDateFormat;
    import java.util.Date;
    import java.util.List;
    
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.security.access.prepost.PreAuthorize;
    import org.springframework.stereotype.Controller;
    import org.springframework.ui.Model;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.ModelAttribute;
    import org.springframework.web.bind.annotation.PostMapping;
    import org.springframework.web.bind.annotation.RequestParam;
    
    import com.sectong.domain.News;
    import com.sectong.domain.User;
    import com.sectong.repository.NewsRepository;
    import com.sectong.repository.UserRepository;
    import com.sectong.service.NewsSearchService;
    import com.sectong.service.UserService;
    
    @Controller
    public class AdminController {
    
        private UserRepository userRepository;
        private NewsRepository newsRepository;
        private NewsSearchService newssearchService;
        private UserService userService;
    
        @Autowired
        public AdminController(NewsRepository newsRepository, UserRepository userRepository, UserService userService,NewsSearchService newssearchService) {
            this.userRepository = userRepository;
            this.newsRepository = newsRepository;
            this.userService = userService;
            this.newssearchService = newssearchService;
        }
    
        /**
         * 管理主界面
         * 
         * @param model
         * @return
         */
        @GetMapping("/admin/")
        public String adminIndex(Model model) {
            model.addAttribute("dashboard", true);
            model.addAttribute("userscount", userRepository.count());
            model.addAttribute("newscount", newsRepository.count());
            return "admin/index";
        }
    
        /**
         * 用户管理
         * 
         * @param model
         * @return
         */
        @GetMapping("/admin/user")
        public String adminUser(Model model) {
            model.addAttribute("user", true);
            return "admin/user";
        }
    
        /**
         * 新闻管理
         * 
         * @param model
         * @return
         * @throws ParseException 
         */
        @GetMapping("/admin/news")
        public String adminNews(Model model) throws ParseException {
            model.addAttribute("news", true);
            Iterable newslist = newsRepository.findAll();
            model.addAttribute("newslist", newslist);
            return "admin/news";
        }
        
    
        
        /**
         * 新闻增加表单
         * 
         * @param model
         * @return
         */
        @PreAuthorize("hasRole('ROLE_ADMIN')")
        @GetMapping("/admin/news/add")
        public String newsAdd(Model model) {
            model.addAttribute("newsAdd", new News());
            return "admin/newsAdd";
        }
    
        /**
         * 新闻修改表单
         * 
         * @param model
         * @return
         */
        @PreAuthorize("hasRole('ROLE_ADMIN')")
        @GetMapping("/admin/news/edit")
        public String newsEdit(Model model, @RequestParam Long id) {
            model.addAttribute("newsEdit", newsRepository.findOne(id));
            return "admin/newsEdit";
        }
    
        /**
         * 新闻修改提交操作
         * 
         * @param news
         * @return
         * @throws IOException 
         * @throws ParseException 
         */
        @PreAuthorize("hasRole('ROLE_ADMIN')")
        @PostMapping("/admin/news/edit")
        public String newsSubmit(@ModelAttribute News news) throws IOException, ParseException {
            SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
            Date d = dateFormat.parse(dateFormat.format(new Date()));
            news.setDatetime(d);
            User user = userService.getCurrentUser();
            news.setUser(user);
            if(null!=news.getId())
            {
                News oldnews = newsRepository.findOne(news.getId());
                newssearchService.updateIndex(oldnews, news);
            }
            else
            {
                newssearchService.createIndex(news);
            }
            newsRepository.save(news);
            return "redirect:/admin/news";
        }
    
        /**
         * 新闻删除操作
         * 
         * @param model
         * @param id
         * @return
         * @throws IOException 
         */
        @PreAuthorize("hasRole('ROLE_ADMIN')")
        @GetMapping("/admin/news/del")
        public String delNews(Model model, @RequestParam Long id) throws IOException {
            News news = newsRepository.findOne(id);
            newssearchService.deleteIndex(news);
            newsRepository.delete(id);
            return "redirect:/admin/news";
        }
    }

4、实现效果

5、代码下载

具体内容在 移动易git代码库 mysql-connector 分支下:

http://git.oschina.net/sectong/yidongyi/tree/f9f8661e27ed8bd41a7249580381fa43482c2f6f

你可能感兴趣的:(全文搜索,lucene,spring,springboot,后台)