3、使用Lucene实现千度搜索

1、新建Web项目

新建一个Web项目，我命名为SearchEngine，然后导入Java包：

除了上篇博客中的Jar包外，我还引入了 IKAnalyzer2012_FF.jar 包和struts2的相关包：

IKAnalyzer：是用来进行中文分词的一个jar包，他会把中文分词一个个合理的词来进行检索；

Struts2:一会儿搜索结果，使用Struts2展示到浏览器中； 

2.准备数据源

我使用linux 命令 wget 爬了一个网站内的一部分html网页，同样将它放在一个纯英文的目录：

3、创建索引

新建一个类CreateIndex:

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.junit.Test;
import org.wltea.analyzer.lucene.IKAnalyzer;
import com.HtmlBeanUtil;
import com.model.HtmlBean;

public class CreateIndex {
    public static final String DATA_DIR="E:/data/engine/www.bjsxt.com";
    public static final String INDEX_DIR="E:/data/engine/index";
    public void createIndex() throws IOException{
        FSDirectory dir = FSDirectory.open(new File(INDEX_DIR));
        // 使用中文分词的jar包进行分词
        IKAnalyzer analyzer = new IKAnalyzer(true);
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
        config.setOpenMode(OpenMode.CREATE_OR_APPEND);
        IndexWriter writer = new IndexWriter(dir, config);
        File file = new File(DATA_DIR);
        Collection<File> files = FileUtils.listFiles(file, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
        for(File f : files){
            // 将原数据源内的内容通过抓取，返回一个实体类方便存储
            HtmlBean hb = HtmlBeanUtil.parseHtml(f);
            if(hb!=null && hb.getTitle()!=null && !hb.getTitle().trim().equals("")){
                Document doc = new Document();
                // 存储三个内容，标题，内容，url （实际上内容可能会更多比如关键字，描述等）
                doc.add(new TextField("title",hb.getTitle(), Store.YES));
                doc.add(new TextField("content",hb.getContent(), Store.YES));
                doc.add(new TextField("url",hb.getUrl(), Store.YES));
                writer.addDocument(doc);
            }
        }
        writer.close();
    }
}

实体HtmlBean和HtmlBeanUtil:

public class HtmlBean {
    private String title;
    private String content;
    private String url;
    public String getTitle() {
        return title;
    }
    public void setTitle(String title) {
        this.title = title;
    }
    public String getContent() {
        return content;
    }
    public void setContent(String content) {
        this.content = content;
    }
    public String getUrl() {
        return url;
    }
    public void setUrl(String url) {
        this.url = url;
    }
}

import java.io.File;
import java.io.IOException;
import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;
import com.model.HtmlBean;
public class HtmlBeanUtil {
    public static HtmlBean parseHtml(File file){
        try {
            Source source = new Source(file);
            Element title = source.getFirstElement(HTMLElementName.TITLE);
            String content = source.getTextExtractor().toString();
            HtmlBean hb = new HtmlBean();
            if(title==null || title.getTextExtractor() == null){
                return null;
            }
            hb.setTitle(title.getTextExtractor().toString());
            hb.setContent(content);
            String path = file.getAbsolutePath();
            String url = "http://"+path.substring(15);
            url = url.replace("\\", "/");
            hb.setUrl("http://"+path.substring(15));
            return hb;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
}

使用单元测试跑一下创建索引的方法，最后会得到这么几个索引数据库文件：

4、创建检索类SearchIndex:

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.model.HtmlBean;
import com.model.Page;

public class SearchIndex {
    public Page search(String keyWord,int pageNum,int pageSize) throws IOException, ParseException, InvalidTokenOffsetsException{
        Directory dir = FSDirectory.open(new File(CreateIndex.INDEX_DIR));
        IndexReader reader = DirectoryReader.open(dir);
        IndexSearcher searcher = new IndexSearcher(reader);
        // 使用中文分词器把用户输入的内容进行分词
        Analyzer analyzer = new IKAnalyzer(true);
        QueryParser parser = new QueryParser(Version.LUCENE_4_9, "title", analyzer);
        Query query = parser.parse(keyWord);
        //format 用来制定要高亮显示的词的样式
        SimpleHTMLFormatter format = new SimpleHTMLFormatter("<font color='red'>","</font>");
        Highlighter high = new Highlighter(format ,new QueryScorer(query));
        // pageNum*pageSize 控制显示的最大条数
        TopScoreDocCollector results = TopScoreDocCollector.create(pageNum*pageSize, false);
        searcher.search(query, results);
        // 检索出来想要的结果的条数，可以实现分页
        TopDocs topDocs = results.topDocs((pageNum-1)*pageSize, pageNum*pageSize);
        Page page = new Page();
        page.setPageNum(pageNum);
        page.setPageSize(pageSize);
        page.setTotalCount(topDocs.totalHits);
        ScoreDoc[] docs = topDocs.scoreDocs;
        List<HtmlBean> list = new ArrayList<HtmlBean>();
        for(ScoreDoc scoreDoc : docs){
            Document document = reader.document(scoreDoc.doc);
            String title = document.get("title");
            String content = document.get("content");
            String url = document.get("url");
            //获取到检索的结果以后，可以使用Highlighter获取高亮效果
            title = high.getBestFragment(analyzer, "title", title);
            content = high.getBestFragment(analyzer, "content", content);
            HtmlBean hb = new HtmlBean();
            hb.setTitle(title);
            hb.setContent(content);
            hb.setUrl(url);
            list.add(hb);
        }
        // 计算记录的总页数
        if(page.getTotalCount() <= pageSize){
            page.setTotalPageCount(1);
        }else{
            if(page.getTotalCount() % pageNum == 0){
                page.setTotalPageCount(page.getTotalCount() / pageSize);
            }else{
                page.setTotalPageCount(page.getTotalCount() / pageSize + 1);
            }
        }
        page.setList(list);
        return page;
    }
}

同时我还用到了一个Page的实体，用来存放并返回查到的结果：

import java.util.List;

public class Page {
    private long totalCount;
    private int pageSize;
    private int pageNum;
    private long totalPageCount;
    private List<HtmlBean> list;
    public long getTotalCount() {
        return totalCount;
    }
    public void setTotalCount(long totalCount) {
        this.totalCount = totalCount;
    }
    public int getPageSize() {
        return pageSize;
    }
    public void setPageSize(int pageSize) {
        this.pageSize = pageSize;
    }
    public int getPageNum() {
        return pageNum;
    }
    public void setPageNum(int pageNum) {
        this.pageNum = pageNum;
    }
    public List<HtmlBean> getList() {
        return list;
    }
    public void setList(List<HtmlBean> list) {
        this.list = list;
    }
    public long getTotalPageCount() {
        return totalPageCount;
    }
    public void setTotalPageCount(long totalPageCount) {
        this.totalPageCount = totalPageCount;
    }
}

5、页面呈现内容

页面呈现内容，由于过于简单就不描述太多了；

Struts.xml和web.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE struts PUBLIC "-//Apache Software Foundation//DTD Struts Configuration 2.1//EN" "http://struts.apache.org/dtds/struts-2.1.dtd">
<struts>
    <constant name="struts.118n.encoding" value="UTF-8"></constant>     
    <constant name="struts.action.extension" value="do"></constant>  
    <package name="pages" namespace="/pages" extends="struts-default">
        <action name="engine_*" class="com.actions.SearchEngineAction" method="{1}">
            <result name="message">/WEB-INF/message.jsp</result>
            <result name="index">/index.jsp</result>
        </action>
    </package>
</struts>

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" id="WebApp_ID" version="3.0">
  <filter>  
   <filter-name>struts2</filter-name>  
   <filter-class>org.apache.struts2.dispatcher.ng.filter.StrutsPrepareAndExecuteFilter</filter-class>  
 </filter>  
 <filter-mapping>  
   <filter-name>struts2</filter-name>  
   <url-pattern>/*</url-pattern>  
 </filter-mapping> 
  <display-name>SearchEngine</display-name>
  <welcome-file-list>
    <welcome-file>index.html</welcome-file>
    <welcome-file>index.htm</welcome-file>
    <welcome-file>index.jsp</welcome-file>
    <welcome-file>default.html</welcome-file>
    <welcome-file>default.htm</welcome-file>
    <welcome-file>default.jsp</welcome-file>
  </welcome-file-list>
</web-app>

 

Action:

import java.io.File;
import java.io.IOException;
import javax.servlet.http.HttpServletRequest;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.struts2.ServletActionContext;
import com.lucene.CreateIndex;
import com.lucene.SearchIndex;
import com.model.Page;

public class SearchEngineAction {
    private int pageNum;
    private String keyword;
    public String create(){
        HttpServletRequest request = ServletActionContext.getRequest();
        try {
            File file = new File(CreateIndex.INDEX_DIR);
            if(file.exists()){
                for(File f : file.listFiles()){
                    f.delete();
                }
                file.delete();
                file.mkdirs();
            }
            CreateIndex createIndex = new CreateIndex();
            createIndex.createIndex();
            request.setAttribute("message", "创建索引完成...");
        } catch (Exception e) {
            e.printStackTrace();
            request.setAttribute("message", e.getMessage());
        }
        return "message";
    }
    
    public String search() throws IOException, ParseException, InvalidTokenOffsetsException{
        HttpServletRequest request = ServletActionContext.getRequest();
        int pageSize = 10;
        if(pageNum < 1){
            setPageNum(1);
        }
        if(keyword!=null && !keyword.trim().equals("")){
            SearchIndex search = new SearchIndex();
            Page page = search.search(keyword, pageNum, pageSize);
            
            request.setAttribute("page", page);
            request.setAttribute("keyword", keyword);
        }
        return "index";
    }

    public int getPageNum() {
        return pageNum;
    }

    public void setPageNum(int pageNum) {
        this.pageNum = pageNum;
    }

    public String getKeyword() {
        return keyword;
    }

    public void setKeyword(String keyword) {
        this.keyword = keyword;
    }
}

页面展示：

<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>千度一下，你就知道</title>
<style type="text/css">
    body{padding-left: 132px;padding-right: 132px;}
    #keyword{width: 521px; height: 20px; padding: 9px 7px;font: 16px arial;  border: 1px solid #b8b8b8; border-bottom: 1px solid #ccc; 
    vertical-align: top; outline: none;  box-shadow: none;}
    .s_btn { width: 100px; height: 40px; color: white;  border: solid 1px #3385ff; font-size:13pt;  letter-spacing: 1px;background: #3385ff;
    border-bottom: 1px solid #2d78f4;outline: medium;  -webkit-appearance: none;  -webkit-border-radius: 0;}
    h4{ font-weight: normal;}
</style>
</head>
<body>
    <form action="/SearchEngine/pages/engine_search.do" method="post">
        <di>
            <input name="keyword" id="keyword" type="text" maxlength="100" autocomplete="off" value="${keyword }" />
            <input type="submit" class="s_btn" value="千度一下" />
        </div>
        <hr />
        <c:if test="${page != null }" >
            <div>千度为您找到相关结果约${page.totalCount}个</div>
            <c:forEach items="${page.list }" var="record">
                <h4><a href="${record.url }" target="_black">${record.title }</a></h4>
                <p>${record.content }
                    <br />
                    <a href="${record.url }" target="_black">${record.url }</a>
                </p>
                
            </c:forEach>
            <div>
             <c:forEach var="i" begin="1" end="${page.totalPageCount }" step="1">
                <a> ${i} </a>
            </c:forEach>
            </div>
        </c:if>
    </form>
</body>
</html>

运行一下看一下效果：

没有搜索的时候：

我们搜索一个比较复杂的词：“尚学堂java官方网站中国媒体报道培训机构” 这些词都是完全合在一起的，看看中文分词器能否给分出来：

 

完全可以，把能够组合的词全显示出来了；

 

说明：在该DEMO中，我爬了尚学堂的网站，但这里没有要给他们打广告的意思哦！纯粹感觉好玩！

 

关键Lucene 根据权重检索，其中有一个算法是TF-IDF算法，详细可查看这篇文章：

http://www.cnblogs.com/biyeymyhjob/archive/2012/07/17/2595249.html

 

小结：在该文中做了一个史上最简陋的搜索引擎，但Lucene的强大是显而已见的，搜索速度也非常快；该小Demo还有很多功能需要补全，比如分页，比如按权重搜索等等等等等...有感兴趣的伙伴可以试着补充全一下，让千度搜索比百度搜索更NB（在梦中YY）; 有需要源码的小伙伴，给我发邮件吧：[email protected]