java调用Linux执行Python爬虫，并将数据存储到elasticsearch中--（java后台代码）

该篇博客主要是java代码，如需相应脚本及java连接elasticsearch工具类代码，请移步到上一篇博客（https://www.cnblogs.com/chenyuanbo/p/9973685.html）

一、创建连接执行Linux脚本工具类

package com.yjlc.platform.utils.Elasticsearch;

import ch.ethz.ssh2.Connection;
import ch.ethz.ssh2.StreamGobbler;

import java.io.*;
/**
 * --------------------------------------------------------------
 * CopyRights(c)2018,YJLC
 * All Rights Reserved
 * 

 * FileName: SingletonUtil.java
 * Description:
 * Author: cyb
 * CreateDate: 2018-11-15
 * --------------------------------------------------------------
 */
public class SingletonUtil {
    //无参构造
    private SingletonUtil(){}
    private volatile static SingletonUtil instance;
    //字符编码默认是utf-8
    public static String  DEFAULTCHART="UTF-8";
    public static Connection conn;
    private String ip;
    private String userName;
    private String userPwd;
    public static Boolean flag=false;
//有参构造
    public SingletonUtil(String ip, String userName, String userPwd) {
        this.ip = ip;
        this.userName = userName;
        this.userPwd = userPwd;
    }

    public SingletonUtil getInstance(String ip, String userName, String userPwd){
        if(instance==null){
            synchronized(SingletonUtil.class){
                //防止多线程多次创建
                if(instance==null){
                    instance=new SingletonUtil(ip,userName, userPwd);
                }
            }
        }
        flag= instance.login();//调用登录方法
        return instance;
    }
    //登录
    public Boolean login(){
        boolean flg=false;
        try {
            System.out.println("进入连接");
            conn = new Connection(ip);
            try {
                conn.connect();//连接
            } catch (IOException e) {
                e.printStackTrace();
            }
            flg=conn.authenticateWithPassword(userName, userPwd);//认证
            if (flg){
                System.out.println("认证成功！");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flg;
    }

    /**
     *@description:纯文本格式返回
     *@author:cyb
     *@date: 2018-11-15 16:56
    *@param: in
    *@param: charset
     *@return: java.lang.String
     */
    public static String processStdout(InputStream in, String charset){
        InputStream    stdout = new StreamGobbler(in);
        StringBuffer buffer = new StringBuffer();;
        try {
            BufferedReader br = new BufferedReader(new InputStreamReader(stdout,charset));
            String line=null;
            while((line=br.readLine()) != null){
                buffer.append(line+"\n");
            }
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return buffer.toString();
    }
}

二、控制层

/**
     *@description:开启爬虫
     *@author:cyb
     *@date: 2018-11-14 15:59
     *@param: id
     *@param: execute
     *@return: java.util.Map
     */
    @RequestMapping("openTask")
    @ResponseBody
    public Map openTask(String id,Boolean execute){
        Map map = new HashMap<>();
        //根据id查询任务详细信息
        BsKnowledgeInfoDTO  knowledgeInfoDTO=  knolegeService.getDataInfoById(id);
        if(execute==true){
            execute=false;
        }else {
            execute=true;
        }
        knowledgeInfoDTO.setExecute(execute);//修改任务的状态（开启、关闭）
        int k = knolegeService.updateDataInfo(knowledgeInfoDTO);
//        StringBuilder url = new StringBuilder(knowledgeInfoDTO.getPath()) ;//爬虫目标路径
        StringBuilder url= new StringBuilder("https://mil.news.sina.com.cn/");
        StringBuilder reptileMethod= new StringBuilder("http://192.168.200.8:8000/news");//爬虫方法http://192.168.200.8:8000/news
        StringBuilder themeid= new StringBuilder("hottopic");//存储索引名称
       //http://192.168.200.8:8000/news?themeid=hottopic&url=https://mil.news.sina.com.cn/history/2018-11-15/doc-ihmutuec0443667.shtml
        StringBuilder path =reptileMethod.append("?").append("themid=").append(themeid).append("&").append("url=").append(url);
        String ip="192.168.200.8";//Linux 路径
        String userName ="root";
        String userPwd ="yjlc20148";
        int w = knolegeService.reptile(path.toString(),ip,userName,userPwd);
        if(w==200){
            map.put("code",200);
            map.put("message","爬虫成功！");
        }else if(w==206){
            map.put("code",206);
            map.put("message","连接失败！");
        }
        return map;
    }

三、service层（此处省略了service接口层）

/**
 *@description: 爬虫
 *@author:cyb
 *@date: 2018-11-15 20:52
*@param: path 爬虫方法路径+ES存储索引+爬虫目标url合集
*@param: ip 连接ip地址
*@param: userName ：用户名
*@param: userPwd：用户密码
 *@return: int
 */
@Override
public int reptile(String path,String ip,String userName,String userPwd) {
    SingletonUtil singletonUtil = new SingletonUtil("192.168.200.8", "root","yjlc20148");
    singletonUtil.getInstance(ip, userName,userPwd);
    Boolean b =SingletonUtil.flag;//看是否连接成功
    if(b==true){
        System.out.println("=====第一个步骤=====");
        Session session= null;//打开一个会话
        try {
            session = singletonUtil.conn.openSession();
            session.execCommand("sh /opt/zc/linux_sina.sh");//执行命令
        } catch (IOException e) {
            e.printStackTrace();
        }
        //TODO:多条命令
        String result=singletonUtil.processStdout(session.getStdout(),singletonUtil.DEFAULTCHART);
        //如果为得到标准输出为空，说明脚本执行出错了
        if(StringUtils.isBlank(result)){
            System.out.println("脚本出错");
           result=singletonUtil.processStdout(session.getStderr(),singletonUtil.DEFAULTCHART);
        }
        System.out.println("第一个步骤脚本运行成功"+result);
        ConnectNetworkUtil connectNetworkUtil = new ConnectNetworkUtil();
        connectNetworkUtil.ConnectNetwork(path);
        System.out.println("采集成功！");
        session.close();//关闭session
        singletonUtil.conn.close();//爬虫关闭连接
        return 200;//爬虫成功
    }else {
        return 206;//连接失败
    }

}

 

以上代码已省略了service接口层和java连接elasticsearch工具类（上一篇博客中已写到），以上代码仅供参考，若代码中有不合理或者不规范的地方，请各位指出，技术在于交流！

 

转载于:https://www.cnblogs.com/chenyuanbo/p/9973769.html