娱乐头条-06IndexWrite

案例：将数据库中的新闻定时写入到索引库中

1、pom文件

  4.0.0

  com.xj
  IndexWrite
  1.0.0
  war

  
    5.0.0.RELEASE
  

  

    
      org.springframework
      spring-context
      ${project.spring.version}
    

    
      org.springframework
      spring-tx
      ${project.spring.version}
    

    
      org.springframework
      spring-jdbc
      ${project.spring.version}
    

    
      org.springframework
      spring-web
      ${project.spring.version}
    
    
      org.springframework
      spring-webmvc
      ${project.spring.version}
    


    
    
      com.alibaba
      druid
      1.1.10
    
    
      mysql
      mysql-connector-java
      5.1.38
    

    
      org.mybatis
      mybatis
      3.4.6
    

    
      org.mybatis
      mybatis-spring
      1.3.2
    

    
      log4j
      log4j
      1.2.17
    
    
      org.slf4j
      slf4j-api
      1.7.25
    
    
      org.slf4j
      slf4j-log4j12
      1.7.25
    


    
      redis.clients
      jedis
      3.0.1
    


    
      org.apache.solr
      solr-solrj
      4.10.2
    

    
      junit
      junit
      4.12
    
  


  
    
      
        org.apache.maven.plugins
        maven-compiler-plugin
        3.7.0
        
          1.8
          1.8
          UTF-8
        
      
    
  

2、实体类

package com.xj.bean;

import org.apache.solr.client.solrj.beans.Field;

import java.util.Date;

public class News {
    @Field
    private Integer id;
    @Field
    private String title;
    @Field
    private String intro;
    @Field
    private String source;
    @Field
    private String vurl;
    @Field
    private Date publishTime;

    @Override
    public String toString() {
        return "News{" +
                "id=" + id +
                ", title='" + title + '\'' +
                ", intro='" + intro + '\'' +
                ", source='" + source + '\'' +
                ", vurl='" + vurl + '\'' +
                ", publishTime=" + publishTime +
                '}';
    }

    public Integer getId() {
        return id;
    }

    public void setId(Integer id) {
        this.id = id;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getIntro() {
        return intro;
    }

    public void setIntro(String intro) {
        this.intro = intro;
    }

    public String getSource() {
        return source;
    }

    public void setSource(String source) {
        this.source = source;
    }

    public String getVurl() {
        return vurl;
    }

    public void setVurl(String vurl) {
        this.vurl = vurl;
    }

    public Date getPublishTime() {
        return publishTime;
    }

    public void setPublishTime(Date publishTime) {
        this.publishTime = publishTime;
    }
}

3、工具类

package com.xj.util;

import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;

public class JedisUtil {
    private static JedisPool jedisPool;
    //静态代码块:
    static {
        JedisPoolConfig config = new JedisPoolConfig();
        config.setMaxTotal(50);
        config.setMaxIdle(20); // 最大的闲时连接
        config.setMinIdle(5); // 最小的闲时连接
        jedisPool = new JedisPool(config,"127.0.0.1",6379);
    }
    //获取连接
    public static Jedis getJedis(){
        return jedisPool.getResource();
    }
}

4、mapper

package com.xj.mapper;


import com.xj.bean.News;

import java.util.List;

public interface NewsMapper {
    List list();
    List getGreaterId(Integer id);
    int getMaxId(Integer id);
}

5、service

package com.xj.service;

public interface INewsService {
    void writeIndex();
}

package com.xj.service.impl;

import com.xj.bean.News;
import com.xj.mapper.NewsMapper;
import com.xj.service.INewsService;
import com.xj.util.JedisUtil;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import redis.clients.jedis.Jedis;

import java.io.IOException;
import java.util.Date;
import java.util.List;


@Service
public class NewsServiceImpl implements INewsService {

    @Autowired
    private NewsMapper mapper;
    @Autowired
    private SolrServer server;

    public void writeIndex(){

        //把所有的新闻写入到索引库
        Integer maxId = 0;

        //循环,下一次循环时,都需要获取上次循环的最大的id值
        while(true){
            Jedis jedis = JedisUtil.getJedis();
            String res = jedis.get("spider:maxId");
            jedis.close();

            maxId = res == null ? 0 : Integer.parseInt(res);

            List list = mapper.getGreaterId(maxId);

            System.out.println(list);

            if(list==null || list.size()<=0){
                break;
            }

            //调整更新日期的值
            for (News news : list) {
                Date updateTime = news.getPublishTime();
                long time = updateTime.getTime()+8*60*60*1000;
                news.setPublishTime(new Date(time));
            }

            //写入到索引库
            try {
                server.addBeans(list);
                server.commit();

            } catch (SolrServerException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }

            int listSize = list.size();
            News news = list.get(listSize - 1);
            maxId = news.getId();

            jedis = JedisUtil.getJedis();
            jedis.set("spider:maxId",maxId+"");
            jedis.close();

        }
    }
}

6、mapper映射文件

    
        
        
        
        
        
        
    

    
        select * from news
    

    
        select * from news where id>#{id} order by id asc limit 0,100
    

    
        select max(t.id) from(select id from news where id>#{id} limit 0,100) t
    

7、applicationContext

8、db

jdbc.driverClassName=com.mysql.jdbc.Driver
jdbc.url=jdbc:mysql:///spider
jdbc.username=root
jdbc.password=root

9、web.xml

  


  
    org.springframework.web.context.ContextLoaderListener
  

  
    contextConfigLocation
    classpath:applicationContext.xml
  

10、app

package com.xj;


import org.junit.Test;
import org.springframework.context.support.AbstractApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class App {
    @Test
    public void testTask() throws Exception{
        //启动Spring容器
        AbstractApplicationContext context = new
                ClassPathXmlApplicationContext("applicationContext.xml");
        context.start();
       // System.in.read();
    }
}

11、IndexWrite
 

package com.xj;

import com.xj.service.INewsService;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

@Component
public class IndexWrite {
    //测试是否添加成功
    public static void main(String[] args) {
       ApplicationContext context = new ClassPathXmlApplicationContext("applicationContext.xml");
       INewsService service = context.getBean(INewsService.class);
       service.writeIndex();
    }

    //这个只是测试，测试计时器是否添加成功
    @Scheduled(cron = "0 0/10 * * * ?")
    public void taskMethod(){
        System.out.println("方法执行了");
    }


    //真正执行写入的时使用到的方法
    //每隔十分钟将数据库中的新闻数据添加到索引库
    @Scheduled(cron = "0 0/10 * * * ?")
    public void write(){
        ApplicationContext context = new ClassPathXmlApplicationContext("applicationContext.xml");
        INewsService service = context.getBean(INewsService.class);
        service.writeIndex();
    }

}