spring boot操作hbase2.0.5

最近项目使用到hbase当做数据库存储数据,经过多番网上搜索查找对hbase操作的功能写成了通用方法,希望对大家有所帮助,后期有新的通用功能会实时更新的,通用方法不采用springboot集成的hbase,使用的事原生hbase方法。
开发环境
系统:windows
JDK:1.8
开发工具:Intellij IDEA
虚拟机:vmware
三台linux节点: 172.162.15.111、172.162.15.112、172.162.15.113
Linux:CentOS 7 迷你版 CentOS-7-x86_64-Minimal-1708.iso
Hadoop: Hadoop-3.1.2
Hbase : hbase-2.0.5
Zookeeper: zookeeper-3.4.1
第二步:配置windows系统host,根据您的需求,这一步可有可无
#虚拟机访问地址
172.162.15.111 davidone
172.162.15.112 davidtwo
172.162.15.113 davidthree

第三步:项目结构配置pom.xml


            org.springframework.boot
            spring-boot-configuration-processor
            true


            org.apache.hbase
                hbase-client
            2.0.5
            
                
                    org.slf4j
                    slf4j-log4j12
                
                
                    log4j
                    log4j
                
                
                    javax.servlet
                    servlet-api
                
            
  

application.yml 这个配置以增量的方式添加到文件

logging:
  config: classpath:Logback.xml
bigdata:
  config:
    hBaseConfigMap: {hbase.zookeeper.quorum : 'davidone:2181,davidtwo:2181,davidthree:2181'}

日志配置文件:Logback.xml



    
    
    
    
    
    
    
    
        
            ${PATTERN}
            UTF-8
        
    
    
    
    
    
        
            ${OPEN_FILE_PATH}/debug/topic_file_debug.%d{yyyy-MM-dd}-%i.log
            180
            
                100MB
            
        
        true
        
            %d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
            utf-8
        
        
            DEBUG
            ACCEPT
            DENY
        
    
    
    
        
            ${OPEN_FILE_PATH}/info/topic_file_sync.%d{yyyy-MM-dd}-%i.log
            180
            
                100MB
            
        
        true
        
            %d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
            utf-8
        
        
            INFO
            ACCEPT
            DENY
        
    
    
    
        
            ${OPEN_FILE_PATH}/error/topic_file_error.%d{yyyy-MM-dd}-%i.log
            180
            
                100MB
            
        
        true
        
            %d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
            utf-8
        
        
            ERROR
            ACCEPT
            DENY
        
    

    
    
    
        
        
        
        
    

第四步:核心代码

BigDataConfig.java 配置文件信息

@Component()
@ConfigurationProperties(prefix = "bigdata.config")
public class BigDataConfig {

private Map hBaseConfigMap;

    public Map gethBaseConfigMap() {
        return hBaseConfigMap;
    }

    public void sethBaseConfigMap(Map hBaseConfigMap) {
        this.hBaseConfigMap = hBaseConfigMap;
    }
}

HBaseRepository.java 通用操作hbase数据代码,包含了通用的二级搜索

@Component
public class HBaseRepository{

    private Logger logger = LoggerFactory.getLogger(this.getClass());

    @Autowired
    private BigDataConfig bigDataConfig;

    private static Configuration configuration = HBaseConfiguration.create();
    //设置连接池
    private static ExecutorService executorServicePoolSize = Executors.newScheduledThreadPool(20);
    private ThreadLocal connectionThreadLocal = new ThreadLocal<>();
    private ThreadLocal adminThreadLocal = new ThreadLocal<>();
    private static Connection connection = null;
    private static Admin admin = null;
    private final byte[] POSTFIX = new byte[] { 0x00 };
    /**
     * 加载HBase配置信息
     */
    private void initConfigurationInfo(){
        Map hBaseConfigMap = bigDataConfig.gethBaseConfigMap();
        if(hBaseConfigMap.size() == 0){
            logger.debug(MessageFormat.format("HBase配置信息初始化失败:{0}",JSON.toJSONString(hBaseConfigMap)));
        }else{
            for (Map.Entry confEntry : hBaseConfigMap.entrySet()) {
                configuration.set(confEntry.getKey(), confEntry.getValue());
            }
        }
    }

    /**
     * 初始化HBase client admin
     *
     */
    private  void initHBaseClientAdmin(){
        try{
            admin = adminThreadLocal.get();
            if(admin == null && connection != null){
                admin = connection.getAdmin();
                adminThreadLocal.set(admin);
            }else{
                logger.debug(MessageFormat.format("创建hBase connection连接 失败:{0}",connection));
            }
        }catch (Exception e){
            logger.error(MessageFormat.format("初始化hBase client admin 客户端管理失败:错误信息:{0}",e));
        }
    }

    /**
     * 初始化HBase资源
     */
    public void initRepository(){
        try{
            initConfigurationInfo();
            connection = connectionThreadLocal.get();
            if(connection == null){
                connection = ConnectionFactory.createConnection(configuration, executorServicePoolSize);
                connectionThreadLocal.set(connection);
            }
            initHBaseClientAdmin();
        }catch (Exception e){
            logger.error(MessageFormat.format("创建hBase connection 连接失败:{0}",e));
            e.printStackTrace();
        }finally {
            close(admin,null,null);
        }

    }

    /**
     * 获取table
     * @param tableName 表名
     * @return Table
     */
    private Table getTable(String tableName) {
        try {
            return connection.getTable(TableName.valueOf(tableName));
        }catch (Exception e){
            e.printStackTrace();
            return  null;
        }
    }

    /**
     * 同时创建多张数据表
     * @param tableMap  数据表 map<表名,列簇集合>
     * @return
     */
    public boolean createManyTable(Map> tableMap){
        try{
            if(admin != null){
                for (Map.Entry> confEntry : tableMap.entrySet()) {
                    createTable(confEntry.getKey(), confEntry.getValue());
                }
            }
        }catch (Exception e){
            logger.error(MessageFormat.format("创建多个表出现未知错误:{0}",e.getMessage()));
            e.printStackTrace();
            return false;
        }finally {
            close(admin,null,null);
        }
        return true;
    }


    /**
     * 创建hbase表和列簇
     * @param tableName   表名
     * @param columnFamily  列簇
     * @return  1:创建成功;0:创建出错;2:创建的表存在
     */
    public int createOneTable (String tableName,String... columnFamily){
        try{
            //创建表,先查看表是否存在,然后在删除重新创建
            if(admin != null){
                return createTable(tableName,Arrays.asList(columnFamily));
            }else{
                logger.error("admin变量没有初始化成功");
                return 0;
            }
        }catch (Exception e){
            logger.debug(MessageFormat.format("创建表失败:{0},错误信息是:{1}",tableName,e.getMessage()));
            e.printStackTrace();
            return 0;
        }finally {
            close(admin,null,null);
        }
    }

    /**
     *
     * @param tableName
     * @param columnFamily
     * @return
     * @throws Exception
     */
    private int createTable(String tableName,List columnFamily) throws Exception {
        if(admin.tableExists(TableName.valueOf(tableName))){
            logger.debug(MessageFormat.format("创建HBase表名:{0} 在HBase数据库中已经存在",tableName));
            return 2;
        }else{
            List familyDescriptors =new ArrayList<>(columnFamily.size());
            for(String column : columnFamily){
                familyDescriptors.add(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(column)).build());
            }
            TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
                    .setColumnFamilies(familyDescriptors).build();
            admin.createTable(tableDescriptor);
            logger.info(MessageFormat.format("创建表成功:表名:{0},列簇:{1}",tableName,JSON.toJSONString(columnFamily)));
            return 1;
        }
    }


    /**
     * 插入or 更新记录(单行单列族-多列多值)
     * @param tableName          表名
     * @param row           行号  唯一
     * @param columnFamily  列簇名称
     * @param columns       多个列
     * @param values        对应多个列的值
     */
    public boolean insertManyColumnRecords(String tableName,String row,String columnFamily,List columns,List values){
        try{
            Table table = getTable(tableName);
            Put put = new Put(Bytes.toBytes(row));
            for(int i = 0; i < columns.size(); i++){
                put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columns.get(i)),Bytes.toBytes(values.get(i)));
                table.put(put);
            }
            logger.info(MessageFormat.format("添加单行单列族-多列多值数据成功:表名:{0},列名:{1},列值:{2}",tableName, JSON.toJSONString(columns),JSON.toJSONString(values)));
            return  true;
        }catch (Exception e){
            e.printStackTrace();
            logger.debug(MessageFormat.format("添加单行单列族-多列多值数据失败:表名:{0};错误信息是:{1}",tableName,e.getMessage()));
            return false;
        }
    }


    /**
     * 插入or更新记录(单行单列族-单列单值)
     * @param tableName          表名
     * @param row           行号  唯一
     * @param columnFamily  列簇名称
     * @param column       列名
     * @param value       列的值
     */
    public boolean insertOneColumnRecords(String tableName,String row,String columnFamily,String column,String value){
        try{
            Table table = getTable(tableName);
            Put put = new Put(Bytes.toBytes(row));
            put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(column),Bytes.toBytes(value));
            table.put(put);
            logger.info(MessageFormat.format("添加单行单列族-单列单值数据成功:表名:{0},列名:{1},列值:{2}",tableName,column,value));
            return true;
        }catch (Exception e){
            logger.debug(MessageFormat.format("添加单行单列族-单列单值数据失败:表名:{0},错误信息是:{1}",tableName,e.getMessage()));
            e.printStackTrace();
            return false;
        }
    }

    /**
     * 根据行号删除表中一条记录
     * @param tableName      表名
     * @param rowNumber 行号
     */
    public boolean deleteDataByRowNumber(String tableName,String rowNumber){
       try{
           Table table = getTable(tableName);
           Delete delete = new Delete(Bytes.toBytes(rowNumber));
           table.delete(delete);
           logger.info(MessageFormat.format("根据行号删除表中记录成功:表名:{0},行号:{1}",tableName,rowNumber));
           return true;
       }catch (Exception e){
            e.printStackTrace();
           logger.debug(MessageFormat.format("根据行号删除表中记录失败:表名:{0},行号:{1}",tableName,rowNumber));
            return false;
       }
    }


    /**
     * 删除列簇下所有数据
     * @param tableName      表名
     * @param columnFamily  列簇
     */
    public boolean deleteDataByColumnFamily(String tableName,String columnFamily){
        try{
            if(!admin.tableExists(TableName.valueOf(tableName))){
                logger.debug(MessageFormat.format("根据行号和列簇名称删除这行列簇相关的数据失败:表名不存在:{0}",tableName));
                return false;
            }
            admin.deleteColumnFamily(TableName.valueOf(tableName),Bytes.toBytes(columnFamily));
            logger.info(MessageFormat.format("删除该表中列簇下所有数据成功:表名:{0},列簇:{1}",tableName,columnFamily));
            return true;
        }catch (Exception e){
            e.printStackTrace();
            logger.debug(MessageFormat.format("删除该表中列簇下所有数据失败:表名:{0},列簇:{1},错误信息:{2}",tableName,columnFamily,e.getMessage()));
            return false;
        }
    }

    /**
     * 删除指定的列 ->删除最新列,保留旧列。
     * 如 相同的rowkey的name列数据 提交两次数据,此方法只会删除最近的数据,保留旧数据
     * @param tableName          表名
     * @param rowNumber     行号
     * @param columnFamily  列簇
     * @param cloumn        列
     */
    public  boolean deleteDataByColumn(String tableName,String rowNumber,String columnFamily,String cloumn){
        try{
            if(!admin.tableExists(TableName.valueOf(tableName))){
                logger.debug(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列失败:表名不存在:{0}",tableName));
                return false;
            }
            Table table = getTable(tableName);
            Delete delete = new Delete(rowNumber.getBytes());
            delete.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(cloumn));
            table.delete(delete);
            logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列成功:表名:{0},行号:{1},列簇:{2},列:{3}",tableName,rowNumber,columnFamily,cloumn));
            return true;
        }catch (Exception e){
            e.printStackTrace();
            logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列失败:表名:{0},行号:{1},列簇:{2},列:{3},错误信息:{4}",tableName,rowNumber,columnFamily,cloumn,e.getMessage()));
            return false;
        }
    }

    /**
     * 删除指定的列 ->新旧列都会删除
     * @param tableName          表名
     * @param rowNumber     行号
     * @param columnFamily  列簇
     * @param cloumn        列
     */
    public  boolean deleteDataByAllcolumn(String tableName,String rowNumber,String columnFamily,String cloumn){
        try{
            if(!admin.tableExists(TableName.valueOf(tableName))){
                logger.debug(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除失败:表名不存在:{0}",tableName));
                return false;
            }
            Table table = getTable(tableName);

            Delete delete = new Delete(rowNumber.getBytes());
            delete.addColumns(Bytes.toBytes(columnFamily),Bytes.toBytes(cloumn));
            table.delete(delete);
            logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除成功:表名:{0},行号:{1},列簇:{2},列:{3}",tableName,rowNumber,columnFamily,cloumn));
            return true;
        }catch (Exception e){
            e.printStackTrace();
            logger.error(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除失败:表名:{0},行号:{1},列簇:{2},列:{3},错误信息:{4}",tableName,rowNumber,columnFamily,cloumn,e.getMessage()));
            return false;
        }
    }

    /**
     * 删除表
     * @param tableName 表名
     */
    public  boolean  deleteTable(String tableName){
        try{
            TableName table = TableName.valueOf(tableName);
            if(admin.tableExists(table)){
                //禁止使用表,然后删除表
                admin.disableTable(table);
                admin.deleteTable(table);
            }
            logger.info(MessageFormat.format("删除表成功:{0}",tableName));
            return true;
        }catch (Exception e){
            e.printStackTrace();
            logger.debug(MessageFormat.format("删除表失败:{0},错误信息是:{1}",tableName,e.getMessage()));
            return false;
        }finally {
            close(admin,null,null);
        }
    }


    /**
     * 查询所有表名
     */
    public List getAllTableNames(){
        List resultList = new ArrayList<>();
        try {
            TableName[] tableNames = admin.listTableNames();
            for(TableName tableName : tableNames){
                resultList.add(tableName.getNameAsString());
            }
            logger.info(MessageFormat.format("查询库中所有表的表名成功:{0}",JSON.toJSONString(resultList)));
        }catch (IOException e) {
            logger.error("获取所有表的表名失败",e);
        }finally {
            close(admin,null,null);
        }
        return resultList;
    }


    /**
     * 根据表名和行号查询数据
     * @param tableName  表名
     * @param rowNumber 行号
     * @return
     */
    public Map selectOneRowDataMap(String tableName,String rowNumber){
        Map resultMap = new HashMap<>();
        Get get = new Get(Bytes.toBytes(rowNumber));
        Table table = getTable(tableName);
        try{
            Result result = table.get(get);
            if(result !=null && !result.isEmpty()){
                for(Cell cell : result.listCells()){
                    resultMap.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                            Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())
                    );
                }
            }
            logger.info(MessageFormat.format("根据表名和行号查询数据:表名:{0},行号:{1},查询结果:{2}",tableName,rowNumber,JSON.toJSONString(resultMap)));
        }catch (Exception e){
            e.printStackTrace();
            logger.debug(MessageFormat.format("根据表名和行号查询数据失败:表名:{0},行号:{1},错误信息:{2}",tableName,rowNumber,e.getMessage()));
        }finally {
            close(null,null,table);
        }
        return resultMap;
    }


    /**
     * 根据不同条件查询数据
     * @param tableName      表名
     * @param columnFamily   列簇
     * @param queryParam     过滤列集合   ("topicFileId,6282")=>("列,值")
     * @param regex          分隔字符
     * @param bool           查询方式:and 或 or | true : and ;false:or
     *
     * @return
     */
    public List> selectTableDataByFilter(String tableName,String columnFamily,List queryParam,String regex,boolean bool){
        Scan scan = new Scan();
        Table table = getTable(tableName);
        FilterList filterList = queryFilterData(columnFamily, queryParam, regex, bool);
        scan.setFilter(filterList);
        return queryData(table,scan);
    }

    /**
     * 分页的根据不同条件查询数据
     * @param tableName         表名
     * @param columnFamily      列簇
     * @param queryParam        过滤列集合   ("topicFileId,6282")=>("列,值")
     * @param regex             分隔字符
     * @param bool              查询方式:and 或 or | true : and ;false:or
     * @param pageSize          每页显示的数量
     * @param lastRow           当前页的最后一行
     * @return
     */
    public List> selectTableDataByFilterPage(String tableName,String columnFamily,List queryParam,String regex,boolean bool,int pageSize,String lastRow){
        Scan scan = new Scan();
        Table table = getTable(tableName);
        FilterList filterList = queryFilterData(columnFamily, queryParam, regex, bool);
        FilterList pageFilterList = handlePageFilterData(scan, pageSize, lastRow);
        pageFilterList.addFilter(filterList);
        scan.setFilter(pageFilterList);
        return queryData(table,scan);
    }

    /**
     * 处理分页数据
     * @param scan          过滤的数据
     * @param pageSize      每页显示的数量
     * @param lastRowKey    当前页的最后一行(rowKey)
     * @return
     */
    private FilterList handlePageFilterData(Scan scan, int pageSize,String lastRowKey){
        Filter pageFilter = new PageFilter(pageSize);
        FilterList pageFilterList = new FilterList();
        pageFilterList.addFilter(pageFilter);
        if(!StringUtils.isEmpty(lastRowKey)){
            byte[] startRow = Bytes.add(Bytes.toBytes(lastRowKey), POSTFIX);
            scan.setStartRow(startRow);
        }
        return pageFilterList;
    }
    /**
     * 处理查询条件
     * @param columnFamily   列簇
     * @param queryParam     过滤列集合   ("topicFileId,6282")=>("列,值")
     * @param regex          分隔字符
     * @param bool           查询方式:and 或 or | true : and ;false:or
     * @return
     */
    private FilterList queryFilterData(String columnFamily,List queryParam,String regex,boolean bool){
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        if(!bool){
            filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
        }
        for(String param: queryParam){
            String[] queryArray = param.split(regex);
            SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(columnFamily), Bytes.toBytes(queryArray[0]), CompareOperator.EQUAL,Bytes.toBytes(queryArray[1]));
            singleColumnValueFilter.setFilterIfMissing(true);
            filterList.addFilter(singleColumnValueFilter);
        }
        return filterList;
    }


    /**
     *  查根据不同条件查询数据,并返回想要的单列 =>返回的列必须是过滤中存在
     * @param tableName         表名
     * @param columnFamily      列簇
     * @param queryParam        过滤列集合   ("topicFileId,6282")=>("列,值")
     * @param regex             分隔字符
     * @param column            返回的列
     * @param bool              查询方式:and 或 or | true : and ;false:or
     * @return
     */
    public List> selectColumnValueDataByFilter(String tableName,String columnFamily,List queryParam,String regex,String column,boolean bool){
        Scan scan = new Scan();
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        if(!bool){
            filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
        }
        Table table = getTable(tableName);
        for(String param: queryParam){
            String[] queryArray = param.split(regex);
            SingleColumnValueExcludeFilter singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(columnFamily), Bytes.toBytes(queryArray[0]), CompareOperator.EQUAL,Bytes.toBytes(queryArray[1]));
            filterList.addFilter(singleColumnValueExcludeFilter);
            scan.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(queryArray[0]));
        }
        scan.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(column));
        scan.setFilter(filterList);
        return queryData(table,scan);

    }

    /**
     * 查询表中所有数据信息
     * @param tableName 表名
     * @return
     */
    public List> selectTableAllDataMap(String tableName){
        Table table = getTable(tableName);
        Scan scan = new Scan();
        return queryData(table,scan);
    }

    /**
     * 分页查询表中所有数据信息
     * @param tableName     表名
     * @param pageSize      每页数量
     * @param lastRow       当前页的最后一行
     * @return
     */
    public List> selectTableAllDataMapPage(String tableName,int pageSize,String lastRow){
        Scan scan = new Scan();
        Table table = getTable(tableName);
        FilterList pageFilterList = handlePageFilterData(scan, pageSize, lastRow);
        scan.setFilter(pageFilterList);
        return queryData(table,scan);
    }


    /**
     * 根据表名和列簇查询所有数据
     * @param tableName     表名
     * @param columnFamily  列簇
     * @return
     */
    public List> selectTableAllDataMap(String tableName,String columnFamily){
        Table table = getTable(tableName);
        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes(columnFamily));
        return queryData(table,scan);
    }

    /**
     * 根据相同行号和表名及不同的列簇查询数据
     * @param tableName     表名
     * @param rowNumber     行号
     * @param columnFamily  列簇
     * @return
     */
    public Map selectTableByRowNumberAndColumnFamily(String tableName,String rowNumber,String columnFamily){
        ResultScanner resultScanner = null;
        Map resultMap = new HashMap<>();
        Table table = getTable(tableName);
        try {
            Get get = new Get(Bytes.toBytes(rowNumber));
            get.addFamily(Bytes.toBytes(columnFamily));
            Result result = table.get(get);
            for(Cell cell :result.listCells()){
                resultMap.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                        Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())
                );
            }
            logger.info(MessageFormat.format("根据行号和列簇查询表中数据信息:表名:{0},行号:{1},列簇:{2},查询结果:{3}",tableName,rowNumber,columnFamily,JSON.toJSONString(resultMap)));
        } catch (IOException e) {
            e.printStackTrace();
            logger.info(MessageFormat.format("根据行号和列簇查询表中数据信息:表名:{0},行号:{1},列簇:{2},错误信息:{3}",tableName,rowNumber,columnFamily,e.getMessage()));
        }finally {
            close(null,resultScanner,table);
        }
        return resultMap;
    }

    /**
     * 查询某行中单列数据
     * @param tableName       表名
     * @param rowNumber       行号
     * @param columnFamily    列簇
     * @param column          列
     * @return
     */
    public String selectColumnValue(String tableName,String rowNumber,String columnFamily,String column){
        Table table = getTable(tableName);
        try {
            Get get = new Get(Bytes.toBytes(rowNumber));
            get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
            Result result = table.get(get);
            logger.info(MessageFormat.format("根据表名、行号、列簇、列查询指定列的值:表名:{0},行号:{1},列簇:{2},列名:{3},查询结果:{4}",tableName,rowNumber,columnFamily,column,Bytes.toString(result.value())));
            return Bytes.toString(result.value());
        } catch (IOException e) {
            e.printStackTrace();
            logger.info(MessageFormat.format("根据表名、行号、列簇、列查询指定列的值:表名:{0},行号:{1},列簇:{2},列名:{3},错误信息:{4}",tableName,rowNumber,columnFamily,column,e.getMessage()));
            return "";
        }finally {
            close(null,null,table);
        }
    }

    private List> queryData(Table table,Scan scan){
        ResultScanner resultScanner =null;
        List> resultList = new ArrayList<>();
        try {
            resultScanner = table.getScanner(scan);
            for(Result result : resultScanner){
                logger.info(MessageFormat.format("查询每条HBase数据的行号:{0}",Bytes.toString(result.getRow())));
                Map resultMap = new HashMap<>();
                resultMap.put("rowKey",Bytes.toString(result.getRow()));
                for(Cell cell :result.listCells()){
                    resultMap.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                            Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())
                    );
                }
                resultList.add(resultMap);
            }
            logger.info(MessageFormat.format("查询指定表中数据信息:表名:{0},查询结果:{1}",Bytes.toString(table.getName().getName()),JSON.toJSONString(resultList)));
        } catch (Exception e) {
            e.printStackTrace();
            logger.debug(MessageFormat.format("查询指定表中数据信息:表名:{0},错误信息:{1}",Bytes.toString(table.getName().getName()),e.getMessage()));
        }finally {
            close(null,resultScanner,table);
        }
        return resultList;
    }


    /**
     * 统计表中数据总数
     * @param tableName     表名
     * @return
     */
    public int getTableDataCount(String tableName){
        Table table = getTable(tableName);
        Scan scan = new Scan();
        return queryDataCount(table,scan);

    }

    /**
     * 统计表和列簇数据总数
     * @param tableName       表名
     * @param columnFamily    列簇
     * @return
     */
    public int getTableDataCount(String tableName,String columnFamily){
        Table table = getTable(tableName);
        Scan scan = new Scan();
        scan.addFamily(Bytes.toBytes(columnFamily));
        return queryDataCount(table,scan);
    }


    private int queryDataCount(Table table,Scan scan){
        scan.setFilter(new FirstKeyOnlyFilter());
        ResultScanner resultScanner = null;
        int rowCount = 0;
        try {
            resultScanner = table.getScanner(scan);
            for(Result result : resultScanner){
                rowCount += result.size();
            }
            logger.info(MessageFormat.format("统计全表数据总数:表名:{0},查询结果:{1}",Bytes.toString(table.getName().getName()),rowCount));
            return rowCount;
        } catch (Exception e) {
            e.printStackTrace();
            logger.debug(MessageFormat.format("查询指定表中数据信息:表名:{0},错误信息:{1}",Bytes.toString(table.getName().getName()),e.getMessage()));
            return rowCount;
        }finally {
            close(null,resultScanner,table);
        }
    }

    /**
     * 关闭流
     */
    private void close(Admin admin, ResultScanner rs, Table table){
        if(admin != null){
            try {
                admin.close();
            } catch (IOException e) {
                logger.error("关闭Admin失败",e);
            }
        }
        if(rs != null){
            rs.close();
        }
        if(table != null){
            try {
                table.close();
            } catch (IOException e) {
                logger.error("关闭Table失败",e);
            }
        }
    }
}

测试HBaseRepository功能

@RunWith(SpringRunner.class)
@SpringBootTest
public class HBaseUtilsTest {

    @Autowired
    private HBaseRepository hBaseRepository;
	
    @Test
    public void initTable(){
        hBaseRepository.initRepository();
        Map> tableMap = new HashMap<>();
        tableMap.put("david_topic:actionFlow",Arrays.asList("info","logs"));
        tableMap.put("david_topic:recording",Arrays.asList("info","logs"));
        tableMap.put("david_topic:syncLog",Arrays.asList("info","logs"));
        tableMap.put("david_topic:uploadVersion",Arrays.asList("info","logs"));
        hBaseRepository.createManyTable(tableMap);
    }

    @Test
    public void createOneTable(){
        //在指定命名空间创建表
        hBaseRepository.createOneTable("david_topic:topictest","info","log");
    }
    @Test
    public void deleteTable(){
        hBaseRepository.deleteTable("david_topic:topictest");
    }
    @Test
    public void insertManyColumnRecords(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        String columnFamily = "info";
        List columns = Arrays.asList("id", "title", "poform", "dept", "level", "createUser");
        List values = Arrays.asList("105155", "瑞风S4品牌临时页面", "Mobile", "营销中心", "普通", "王丽");
        hBaseRepository.insertManyColumnRecords(tableName, rowNumber, columnFamily, columns, values);
    }
    @Test
    public  void selectTableByRowNumberAndColumnFamily(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        String columnFamily = "info";
        hBaseRepository.selectTableByRowNumberAndColumnFamily(tableName,rowNumber,columnFamily);
    }

    @Test
    public void getAllTableNames(){
        hBaseRepository.getAllTableNames();
    }

    @Test
    public void insertOneColumnRecords(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        String columnFamily = "info";
        hBaseRepository.insertOneColumnRecords(tableName, rowNumber, columnFamily, "delete", "否");
    }
    @Test
    public void deleteDataBycolumn(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        String columnFamily = "info";
        String column = "delete";
        hBaseRepository.deleteDataByColumn(tableName,rowNumber,columnFamily,column);
    }

    @Test
    public void getTableDataCount(){
        String tableName = "david_topic:topictest";
        hBaseRepository.getTableDataCount(tableName,"log");
    }

    @Test
    public void deleteDataByRowNumber(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        hBaseRepository.deleteDataByRowNumber(tableName, rowNumber);
    }


    @Test
    public void selectTableAllDataMap(){
        String tableName = "david_topic:topictest";
        hBaseRepository.selectTableAllDataMap(tableName);
    }

    @Test
    public void selectTableAllDataMapColumnFamily(){
        String tableName = "david_topic:topictest";
        String columnFamily = "log";
        hBaseRepository.selectTableAllDataMap(tableName,columnFamily);
    }

    @Test
    public void selectOneRowDataMap(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        hBaseRepository.selectOneRowDataMap(tableName,rowNumber);
    }

    @Test
    public void selectColumnValue(){
        String tableName = "david_topic:topictest";
        String rowNumber  = "105155";
        String columnFamily = "info";
        String column = "title";
        hBaseRepository.selectColumnValue(tableName,rowNumber,columnFamily,column);
    }


    @Test
    public void deleteDataByRowNumberAndColumnFamily(){
        String tableName = "david_topic:topictest";
        String columnFamily = "info";
        hBaseRepository.deleteDataByColumnFamily(tableName,columnFamily);
    }

    @Test
    public void selectTableDataByFilter(){
        String tableName = "david_topic:topictest";
        List queryParam = Arrays.asList("id,105155","dept,营销中心");
        hBaseRepository.selectTableDataByFilter(tableName, "info", queryParam, ",",true);
    }

    @Test
    public void selectColumnValueDataByFilter(){
        String tableName = "david_topic:topictest";
        List queryParam = Arrays.asList("topicFileId,6282");
        hBaseRepository.selectColumnValueDataByFilter(tableName, "info", queryParam, ",","id",true);
    }

    /**
     * 分页方式一:
     * 全表扫描并返回经过分页后的数据
     */
    @Test
    public void selectTableAllDataMapAllPage(){
        String tableName ="david_topic:topictest";
        int pageSize = 3;
        String key = null;
        int dataCount = pageSize;
        while (dataCount == pageSize) {
            String mobileKey = null;
            if (key != null) {
                mobileKey = key;
            }
            List> result = hBaseRepository.selectTableAllDataMapPage(tableName, pageSize, mobileKey);
            if(result != null && result.size() > 0 ){
                dataCount = result.size();
                key = result.get(dataCount-1).get("rowKey").toString();
                System.out.println(key);
            }else{
                break;
            }
        }
    }

    /**
     * 分页方式二:
     * 返回每页数据,下一页需要入参【上一页最后一位的rowKey】
     */
    @Test
    public void selectTableAllDataMapPage(){
        String tableName ="david_topic:topictest";
        int pageSize = 3;
        List> result = hBaseRepository.selectTableAllDataMapPage(tableName, pageSize, "32956703305764864");
        System.out.println(result);
    }

}

你可能感兴趣的:(spring boot操作hbase2.0.5)