java 线程池管理多线程操作Hbase数据库完整项目

Hbase-site.xml配置文件:

    
        hbase.rootdir
        hdfs://集群1（zhz100）:9000/hbase
    
    
        hbase.cluster.distributed
        true
    
    
        hbase.master
        集群（zhz100）:60000
    
    
        hbase.zookeeper.property.dataDir
        /usr/java/zookeeper3.4.10/temp
    
    
        hbase.zookeeper.quorum
        集群1（zhz100）,集群2,集群3
    
    
        hbase.zookeeper.property.clientPort
        2181
    

配置application.yml:

	server:
	port: 8080
	
	hbase:
	    conf:
	        confMaps:
	        'hbase.zookeeper.quorum' : 'IP地址1:2181,IP地址2:2181'

pom.xml 引入的依赖:

   
            org.springframework.boot
            spring-boot-starter-web
    
    
    org.springframework.boot
    spring-boot-starter-logging
    
    
        commons-io
        commons-io
        2.2
    
    
        org.springframework.boot
        spring-boot-starter-test
        test
    

    
        org.apache.hbase
        hbase-client
        1.2.0
        
            
                org.slf4j
                slf4j-log4j12
            
        
    
    
        org.springframework.data
        spring-data-hadoop
        2.5.0.RELEASE
    
    
        org.apache.hadoop
        hadoop-hdfs
        2.5.1
    
    
        org.springframework.data
        spring-data-hadoop-core
        2.4.0.RELEASE
    
    
        org.apache.hbase
        hbase
        1.2.1
        pom
    
    
        ch.ethz.ganymed
        ganymed-ssh2
        build210
    
    
        org.springframework.boot
        spring-boot-starter-web
        
            
            org.springframework.boot
            spring-boot-starter-logging
        
        
    

Test含主函数main：

package com.example.demo;
import org.apache.hadoop.hbase.client.Connection;
import sun.net.ftp.FtpClient;
import sun.net.ftp.FtpProtocolException;

import java.io.*;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;


public class Test {
    FtpClient ftpClient;

/**
 * 连接FTP服务
 * @param url //IP地址
 * @param port//端口号
 * @param username//用户名
 * @param password//密码
 * @return
 */
public static FtpClient connectFTP(String url, int port, String username, String password) {

         //创建ftp
     FtpClient ftp = null;
    try {
        //创建地址
        SocketAddress addr = new InetSocketAddress(url, port);
        //连接
        ftp = FtpClient.create();
        ftp.connect(addr);
        //登陆
        ftp.login(username, password.toCharArray());
        ftp.setBinaryType();

    } catch (FtpProtocolException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return ftp;
}

public static List download(String ftpFile, FtpClient ftp) {
    synchronized (TimerWatch.class) {
        List list = new ArrayList();
        String str = "";
        InputStream is = null;
        BufferedReader br = null;
        try {
            // 获取ftp上的文件
            is = ftp.getFileStream(ftpFile);
            //转为字节流
            br = new BufferedReader(new InputStreamReader(is));
            while ((str = br.readLine()) != null) {
                list.add(str);
            }
            br.close();
        } catch (FtpProtocolException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }
}
    public static void main(String[] args) throws Exception{

	//开启主机1FTP服务器
        FtpClient ftp = connectFTP("主机1IP"端口号,"账号","密码");
	//开启主机1Linux指令
        RemoteExecuteCommand rec = new RemoteExecuteCommand("主机1IP", "账号", "密码"); 
        //连接数据库
        Connection connection=LinkHbase.table();
	//如果出现宕机或者出现异常导致项目停止运行，此段代码可以将未成功插入的数据所在的文件就会保存在work目录中。重启后对work进行解析就会避免数据丢失，此方法只会在启动时执行一次。            
         Sync.Judge(connection,ftp,rec);
        //开启一个线程池，corePoolSize：核心线程数5；maxPoolSize：最大线程 数10； keepAliveTime：线程空闲时间：200微妙；ArrayBlockingQueue是一个基于数组结构的有界阻塞队列，此队列按 FIFO（先进先出）原则对元素进行排序
        ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 10, 200,                 TimeUnit.MILLISECONDS,new ArrayBlockingQueue(5));

        for(int i=0;i<5;i++){
            MyTask myTask = new MyTask(i,connection,ftp,rec);
            executor.execute(myTask);
            System.out.println("线程池中线程数目："+executor.getPoolSize()+"，队列中等待执行的任务数目："+
                    executor.getQueue().size()+"，已执行玩别的任务数目："+executor.getCompletedTaskCount());
        }
        executor.shutdown();
    }
}

Sync类中Judge()方法:

//判断in文件是否有文件执行完成
public static void Judge(Connection connection,FtpClient ftp,RemoteExecuteCommand rec) throws Exception {
	
//Linux命令获取/usr/HDWork/work的文件名字
   String files=rec.execute("cd /usr/HDWork/work ; ls -lrt|sed -n '2, $p'|awk '{print $9}'");
    if(files.equals("") || files==null){
        System.out.println("work目录下没有文件");
    } else {
        System.out.println("work目录有文件");
        String[] strs = files.split("\\n");
        for (int i = 0, len = strs.length; i < len; i++) {
            SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//计算运行时间
            long start, end;
            start = System.currentTimeMillis();
            Date date1 = new Date(start);
            String date1String = formatter.format(date1);

            System.out.println(strs[i].toString());
	//解析work目录下的文件内容，插入到hbase
            ReadIn.add(strs[i].toString(), connection, ftp);                
        //移动文件到back目录下
        Cmd.mvBack(strs[i].toString(), rec);
            end = System.currentTimeMillis();
            Date date2 = new Date(end);
            String date2String = formatter.format(date2);
            System.out.println(Thread.currentThread().getName() + ":执行文件：." + strs[i].toString() + "；开始时间为：" + date1String + "；结束时间为：" + date2String + "；添加成功,共用时" + (end - start)+ "ms。");
                }
    }

ReadIn类中add()方法：

public class ReadIn {
    /*
    *解析work目录下的文件内容，插入到数据库中
    * */
    public static void add(String fileName, Connection connection, FtpClient ftpClient) throws Exception{
    String[] y= { "trademark" , "dr_type" , "service_id" , "bill_month" );
    Table table=table(connection);//判断是否已经存在含有这个表
    List cutstrings;
    System.out.println("/usr/HDWork/work/"+fileName);
    List list=download("/usr/HDWork/work/"+fileName,ftpClient);

    String str;

            for(int i=6;i

/*
* 线程锁判断是否有这个表
* */

public static Table table(Connection connection)throws Exception{
    synchronized (ReadIn.class) {
        Admin admin = connection.getAdmin();
        Table table = null;
        SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
        String systemTime = String.valueOf(df.format(new Date()));//系统日期
        TableName tableNameObj = TableName.valueOf(systemTime);//以系统日期为表的名字
        if (admin.tableExists(tableNameObj)) {//如果表的名字已经存在
            table = connection.getTable(TableName.valueOf(systemTime));//连接这个表
            return table;
        } else {//没有这个表，先创建表，在链接表
            HbaseDemo.createTable(systemTime, "trademark", "dr_type", "service_id")；
            table = connection.getTable(TableName.valueOf(systemTime));
            return table;
        }
    }
}

/*
 * 截取前19位字符后解析的内容
 * */
public   static List cutstring (String Stence) {
    List stringlist = new ArrayList();//用来存储解析出来的元素
    for (int i = 0; i < Stence.length(); i++) {
        if (Stence.charAt(i) == ';') {
            String temp = "";//存储单词
            int wordlength = i;
            while (wordlength < Stence.length() - 1 && Stence.charAt(++wordlength) != ';') {
                temp += Stence.charAt(wordlength);
                //System.out.println(temp);
            }
            stringlist.add(temp);
        }
    }
    return stringlist;
}
/**
 * 取ftp上的文件内容
 * @param ftpFile
 * @param ftp
 * @return
 */
public static List download(String ftpFile, FtpClient ftp) {
    synchronized (ReadIn.class) {
        List list = new ArrayList();
        String str = "";
        InputStream is = null;
        BufferedReader br = null;
        try {
            ftpFile=ftpFile.replace(" ","");
            // 获取ftp上的文件
            is = ftp.getFileStream(ftpFile);
            //转为字节流
            br = new BufferedReader(new InputStreamReader(is));
            while((str=br.readLine())!=null){
                list.add(str);
            }
            System.out.println(list+"-------------");
            br.close();
        }catch (FtpProtocolException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }
}

}

MyTask类：

package com.example.demo;

import org.apache.hadoop.hbase.client.Connection;
import sun.applet.Main;
import sun.net.ftp.FtpClient;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.locks.Lock;
import java.util.logging.FileHandler;
import java.util.logging.Logger;

public class MyTask implements Runnable {
    private int taskNum;
    private Connection connection;
    private Lock lock;
    private FtpClient ftpClient;
    private RemoteExecuteCommand rec;



    public MyTask(int num, Connection connection, FtpClient ftpClient,RemoteExecuteCommand rec) {
        this.taskNum = num;
        this.connection=connection;
        this.ftpClient=ftpClient;
        this.rec=rec;
    }
    
    @Override
    public void run(){

        System.out.println(Thread.currentThread().getName() + "线程的名字");
        final Logger logger=Logger.getLogger(Main.class.toString());//日志
        StringBuffer logPath=new StringBuffer();

        logPath.append("E:\\Logger");     //设置保存路径

        //设置文件名
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        logPath.append("\\"+sdf.format(new Date())+".log");
        //将输出handler加入logger
        try {
            FileHandler fileHandler=new FileHandler(logPath.toString(),true);
            logger.addHandler(fileHandler);
        }catch (IOException e){
            e.printStackTrace();
        }

        try {

            while (true) {
                SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                long start,end;
                start = System.currentTimeMillis();
                Date date1=new Date(start);
                String date1String = formatter.format(date1);


                RemoteExecuteCommand rec=new RemoteExecuteCommand("10.248.56.120", "root","password");
               String aa= rec.execute("ls /usr/HDWork/in");
                if (aa.length() > 0) {


                    //Sync sync = new Sync();//开启静态锁，（静态锁不能封装到方法中，此处尝试错误）
                  // String fileName=sync.Mv();//调用静态锁方法，获取work目录中文件时间最小的文件，将文件名称存放在1.txt；将获取的文件移动到in目录中等待操作,删除1.txt。
                    String fileName=this.Mv();//静态锁方法，获取work目录中文件时间最小的文件名字，将文件名称存放在1.txt；将获取的文件移动到in目录中等待操作，将文件名字返回,。
                    if(fileName==null || fileName.equals("")){
                        System.out.println(Thread.currentThread().getName()+"没有文件");
                    }else {
                        ReadIn.add(fileName, connection,ftpClient);//解析in目录下的文件内容，插入到hbase
                    }
                    if(fileName==null || fileName.equals("")){
                        System.out.println(Thread.currentThread().getName()+"没有文件");
                    }else {
                       if(Cmd.mvBack(fileName,rec)) {
                           end = System.currentTimeMillis();
                           Date date2=new Date(end);
                           String date2String = formatter.format(date2);
                           System.out.println(Thread.currentThread().getName() + ":执行文件：." + fileName + "；开始时间为："+date1String+"；结束时间为："+date2String+"；添加成功,共用时"+(end-start)+"ms。");
                           logger.info(Thread.currentThread().getName() + ":执行文件：." + fileName + "；开始时间为："+date1String+"；结束时间为："+date2String+"；添加成功,共用时"+(end-start)+"ms。");
                        }else {
                            System.out.println("过滤异常");
                            logger.info("数据插入成功，但是移动到back中失败");
                        }
                    }
                } else {

                    System.out.println("run中的=="+Thread.currentThread().getName()+"没有文件");
                    Thread.sleep(1000);
                }
            }
        }catch (Exception e){

            System.out.println("run异常"+e.getMessage());
        }
    }

//此处用到了线程锁，避免多个线程同时操作同一个文件，重复添加
    synchronized public String  Mv(){
        synchronized (MyTask.class) {
            rec.execute("cd /usr/HDWork/in");
            String fileName = rec.execute("cd /usr/HDWork/in ; ls -lrt|sed -n \"2, 1p\"|awk '{print $9}'");
            fileName = fileName.replace("\n", " ");
            String mv = "mv /usr/HDWork/in/" + fileName + " /usr/HDWork/work";
            if (!"".equals(fileName)) {
            rec.execute(mv);
            System.out.println(fileName + "======================");
            return fileName;
        } else {
            return null;
        }
    }
    }
}

LinkHbase类：连接数据库

public class LinkHbase {
    private static Configuration conf = HBaseConfiguration.create();
    private static Admin admin;
    static {
        conf.set("hbase.rootdir", "hdfs://node1:9000/hbase");
        // 设置Zookeeper,直接设置IP地址
        conf.set("hbase.zookeeper.quorum", "集群1,集群2,集群3");
    }
    public static Connection table() throws  Exception{
        Connection connection = ConnectionFactory.createConnection(conf);
        return  connection;
    }
}

RemoteExcuteCommaand类：远程执行Linux

/**
 * 远程执行linux的shell script
 * @author Ickes
 * @since  V0.1
 */		
public class RemoteExecuteCommand {

    //字符编码默认是utf-8
    private static String  DEFAULTCHART="UTF-8";
    private Connection conn;
    private String ip;
    private String userName;
    private String userPwd;

    public RemoteExecuteCommand(String ip, String userName, String userPwd) {
        this.ip = ip;
        this.userName = userName;
        this.userPwd = userPwd;
    }
   //无参方法
    public RemoteExecuteCommand() {
    }
    /**
     * 远程登录linux的主机
     * @author Ickes
     * @since  V0.1
     * @return
     *      登录成功返回true，否则返回false
     */
    public Boolean login(){
        boolean flg=false;
        try {
            conn = new Connection(ip);
            conn.connect();//连接
            flg=conn.authenticateWithPassword(userName, userPwd);//认证
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flg;
    }

/**
 * @author Ickes
 * 远程执行shll脚本或者命令
 * @param cmd
 *      即将执行的命令
 * @return
 *      命令执行完后返回的结果值
 * @since V0.1
 */
public String execute(String cmd){
    String result="";
    try {
        if(login()){
            Session session= conn.openSession();//打开一个会话
            session.execCommand(cmd);//执行命令
            result=processStdout(session.getStdout(),DEFAULTCHART);
            //如果为得到标准输出为空，说明脚本执行出错了
            if(StringUtils.isBlank(result)){
                result=processStdout(session.getStderr(),DEFAULTCHART);
            }
            conn.close();
            session.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return result;
}

/**
 * @author Ickes
 * 远程执行shll脚本或者命令
 * @param cmd
 *      即将执行的命令
 * @return
 *      命令执行成功后返回的结果值，如果命令执行失败，返回空字符串，不是null
 * @since V0.1
 */
public String executeSuccess(String cmd){
    String result="";
    try {
        if(login()){
            Session session= conn.openSession();//打开一个会话
            session.execCommand(cmd);//执行命令
            result=processStdout(session.getStdout(),DEFAULTCHART);
            conn.close();
            session.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return result;
}

/**
 * 解析脚本执行返回的结果集
 * @author Ickes
 * @param in 输入流对象
 * @param charset 编码
 * @since V0.1
 * @return
 *       以纯文本的格式返回
 */

private String processStdout(InputStream in, String charset){
    InputStream    stdout = new StreamGobbler(in);
    StringBuffer buffer = new StringBuffer();;
    try {
        BufferedReader br = new BufferedReader(new InputStreamReader(stdout,charset));
        String line=null;
        while((line=br.readLine()) != null){
            buffer.append(line+"\n");
        }
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    } catch (IOException e) 

        e.printStackTrace();
    }
    return buffer.toString();
}

public static void setCharset(String charset) {
    DEFAULTCHART = charset;
}
public Connection getConn() {
    return conn;
}
public void setConn(Connection conn) {
    this.conn = conn;
}
public String getIp() {
    return ip;
}
public void setIp(String ip) {
    this.ip = ip;
}
public String getUserName() {
    return userName;
}
public void setUserName(String userName) {
    this.userName = userName;
}
public String getUserPwd() {
    return userPwd;
}
public void setUserPwd(String userPwd) {
    this.userPwd = userPwd;
}