Hbase 2.1.5版本的 API操作（HBase工具类封装）

前言

本文着重练习了 Hbase API 操作：

• 获取连接对象（使用线程池），当然，有的朋友也喜欢使用 ThreadLocal类做操作。
• 创建命名空间。
• 创建表。
• 判断表是否存在。
• 插入数据。
• 查找单行数据。
• 查找指定行的指定列。
• 全表扫描。
• 扫描+过滤器。
• 删除指定数据。
• 删除表。

1、环境

java环境

JDK 1.8

Hbase 环境

2.1.5

POM文件

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0modelVersion>

    <groupId>org.fenggroupId>
    <artifactId>hbase-demoartifactId>
    <version>1.0-SNAPSHOTversion>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.pluginsgroupId>
                <artifactId>maven-compiler-pluginartifactId>
                <configuration>
                    <source>8source>
                    <target>8target>
                configuration>
            plugin>
        plugins>
    build>

    <properties>
        <project.build.sourceEncoding>UTF-8project.build.sourceEncoding>
        <maven.compiler.target>1.8maven.compiler.target>
        <maven.compiler.source>1.8maven.compiler.source>
    properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hbasegroupId>
            <artifactId>hbase-clientartifactId>
            <version>2.1.5version>
        dependency>
    dependencies>
project>

Log4j.properties

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=log/hbase.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

项目目录

2、HBaseConfig.java

package org.feng.hbase.config;

/**
 * Created by Feng on 2020/1/13 13:00
 * CurrentProject's name is hbase-demo
 * @author Feng
 */
public interface HbaseConfig {
    /**
     * 主机
     */
    String QUORUM = "chost";

    /**
     * 端口
     */
    String CLIENT_PORT = "2181";

}

3、HbaseDao.java

package org.feng.hbase.util;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.util.Bytes;
import org.feng.hbase.config.HbaseConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;


/**
 * Created by Feng on 2020/1/13 11:35
 * CurrentProject's name is hbase-demo

 *     Hbase API 简单封装
 * @author Feng
 */
public class HbaseDao {

    private static final Logger LOGGER = LoggerFactory.getLogger(HbaseDao.class);
    private static final HbaseDao INSTANCE = new HbaseDao();
    private static volatile Connection connection = null;

    private static volatile Admin admin = null;
    /*
     * 初始化数据：配置信息；获取 connection 对象
     */
    static {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", HbaseConfig.QUORUM);
        configuration.set("hbase.zookeeper.property.clientPort", HbaseConfig.CLIENT_PORT);
        try {
            connection = ConnectionFactory.createConnection(
                    configuration,
                    new ThreadPoolExecutor(
                            20,
                            100,
                            60,
                            TimeUnit.MINUTES,
                            new LinkedBlockingQueue<>(20),
                            new ThreadFactoryImpl("org.feng.hbase.util")
                    )
            );
        } catch (IOException e) {
            LOGGER.error("Create connection or admin error! " + e.getMessage(), e);
        }
    }

    private HbaseDao (){}
    public static HbaseDao getInstance(){
        return INSTANCE;
    }


    /**
     * 初始化命名空间：若命名空间存在则不创建
     * @param namespace 命名空间
     */
    public void createNameSpace(String namespace) throws IOException {
        try{
            admin = connection.getAdmin();
            admin.getNamespaceDescriptor(namespace);
            LOGGER.error("NameSpace {} is exist!", namespace);
        } catch (NamespaceNotFoundException e){
            admin.createNamespace(NamespaceDescriptor.create(namespace).build());
            LOGGER.info("Created namespace: {}", namespace);
        }
    }

    /**
     * 删除表：先禁用再删除
     * @param name 表名
     * @throws IOException io操作
     */
    public void deleteTable(String name) throws IOException {
        TableName tableName = TableName.valueOf(name);
        admin = connection.getAdmin();
        admin.disableTable(tableName);
        admin.deleteTable(tableName);
        LOGGER.info("Deleted table {} !", name);
    }

    /**
     * 创建表：表存在时，先删除再创建；
     * 分区数为默认
     * @param tableName 表名
     * @param columnFamily 列族
     * @throws IOException io操作
     */
    public void createTable(String tableName, String...columnFamily) throws IOException {
        createTable(tableName, 0, columnFamily);
    }


    /**
     * 创建表：表存在时，先删除再创建
     * @param tableName 表名
     * @param regionCount 分区数
     * @param columnFamily 列族
     * @throws IOException io操作
     */
    public void createTable(String tableName, int regionCount, String...columnFamily) throws IOException {
        TableName name = TableName.valueOf(tableName);
        admin = connection.getAdmin();
        // 存在
        if(admin.tableExists(name)){
            LOGGER.error("Table named {} already exist!", name);
            deleteTable(tableName);
        }

        createTableTemplate(name, regionCount, columnFamily);
    }

    /**
     * 表是否存在
     * @param tableName 表名
     */
    public boolean tableExists(String tableName) throws IOException {
        TableName name = TableName.valueOf(tableName);
        return getAdmin().tableExists(name);
    }


    /**
     * 插入数据：单行、单列族 => 多列多值
     * @param tableName 表名
     * @param rowKey 行
     * @param columnFamily 列族
     * @param columns 列
     * @param values 值(与列一一对应)
     */
    public void insertRecords(String tableName, String rowKey, String columnFamily, String[] columns, String[] values) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        Put put = new Put(Bytes.toBytes(rowKey));

        for (int i = 0; i < columns.length; i++) {
            put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
            table.put(put);
        }
    }

    /**
     * 插入数据：单行、单列族 => 单列单值
     * @param tableName 表名
     * @param rowKey 行
     * @param columnFamily 列族
     * @param column 列名
     * @param value 列值
     */
    public void insertRecord(String tableName, String rowKey, String columnFamily, String column, String value) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        Put put = new Put(Bytes.toBytes(rowKey));

        put.addColumn(
                Bytes.toBytes(columnFamily),
                Bytes.toBytes(column),
                Bytes.toBytes(value));
        table.put(put);
    }


    /**
     * 删除一份数据
     * @param tableName 表名
     * @param rowKey 行名
     */
    public void deleteRow(String tableName, String rowKey) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        table.delete(new Delete(rowKey.getBytes()));
    }


    /**
     * 删除单行单列族记录
     * @param tableName 表名
     * @param rowKey 行名
     * @param columnFamily 列族
     */
    public void deleteColumnFamily(String tableName, String rowKey, String columnFamily) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        table.delete(new Delete(rowKey.getBytes()).addFamily(Bytes.toBytes(columnFamily)));
    }

    /**
     * 删除单行单列族单列记录
     * @param tableName 表名
     * @param rowKey 行名
     * @param columnFamily 列族
     * @param column 列
     */
    public void deleteColumn(String tableName, String rowKey, String columnFamily, String column) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        table.delete(new Delete(rowKey.getBytes()).addColumn(Bytes.toBytes(columnFamily),
                Bytes.toBytes(column)));
    }

    public void delete(String tableName, Delete delete) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        table.delete(delete);
    }

    public void deleteList(String tableName, List<Delete> delete) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        table.delete(delete);
    }

    /**
     * 查找一行记录
     * @param tableName 表名
     * @param rowKey 行名
     * @return 结果
     */
    public String selectRow(String tableName, String rowKey) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);

        Result result = table.get(new Get(rowKey.getBytes()));
        StringBuffer sb = new StringBuffer();
        resultToString(sb, result);

        return sb.toString();
    }

    /**
     * 查询单行、单列族、单列的值
     * @param tableName 表名
     * @param rowKey 行名
     * @param columnFamily 列族
     * @param column 列名
     * @return 列值
     */
    public String selectValue(String tableName, String rowKey, String columnFamily, String column) throws IOException {
        Get get = new Get(rowKey.getBytes());
        get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));

        TableName name=TableName.valueOf(tableName);
        Result result = connection.getTable(name).get(get);
        return Bytes.toString(result.value());
    }

    /**
     * 全表扫描
     * @param tableName 表名
     * @see Scan
     */
    public String scanAllRecord(String tableName) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);

        Scan scan = new Scan();
        StringBuffer sb = new StringBuffer();
        try(ResultScanner scanner = table.getScanner(scan)){
            for (Result result : scanner) {
                resultToString(sb, result);
            }
        }

        return sb.toString();
    }

    /**
     * 拼接结果
     */
    private void resultToString(StringBuffer sb, Result result) {
        for (Cell cell : result.rawCells()) {
            sb.append(Bytes.toString(cell.getRowArray())).append("\t")
                    .append(Bytes.toString(cell.getFamilyArray())).append("\t")
                    .append(Bytes.toString(cell.getQualifierArray())).append("\t")
                    .append(Bytes.toString(cell.getValueArray())).append("\n");
        }
    }

    /**
     * 过滤器扫描：参考
     * https://www.cnblogs.com/Transkai/p/10727257.html

     *     该 API 较老，使用时要注意查看过时方法上的注释，找新的 API 使用
     * @param tableName 表名
     * @param filter 过滤器
     */
    public List<Cell> scanByFilter(String tableName, Filter filter) throws IOException {
        List<Cell> resultList = new ArrayList<>();
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);

        Scan scan = new Scan();
        scan.setFilter(filter);
        try(ResultScanner scanner = table.getScanner(scan)){
            for (Result result : scanner) {
                resultList.addAll(Arrays.asList(result.rawCells()));
            }
        }

        return resultList;
    }


    /**
     * 创建表
     * @param tableName 表名
     * @param regionCount 分区数
     * @param columnFamily 列族
     */
    private void createTableTemplate(TableName tableName, int regionCount, String...columnFamily) {
        try {
            admin = connection.getAdmin();
            TableDescriptorBuilder tableBuilder = TableDescriptorBuilder.newBuilder(tableName);
            // 增加列族
            tableBuilder.setColumnFamilies(createColumnFamilyList(columnFamily));

            // 无分区（未指定）
            if(regionCount <= 0){
                admin.createTable(tableBuilder.build());
            } else {
                // 预分区
                byte[][] splitKey = getSplitKeys(regionCount);
                admin.createTable(tableBuilder.build(), splitKey);
            }
            LOGGER.info("Created table named {}", tableName);
        } catch (IOException e) {
            LOGGER.error("Create table error, " + e.getMessage(), e);
        }
    }

    /**
     * 列族描述器：没有内容时，自定义加一个列族 info
     * @param columnFamily 列族名
     * @return 列族描述器
     */
    private List<ColumnFamilyDescriptor> createColumnFamilyList(String...columnFamily){
        List<ColumnFamilyDescriptor> results = new ArrayList<>();
        // 设置默认列族 info
        if(columnFamily == null || columnFamily.length == 0){
            columnFamily = new String[]{"info"};
        }

        for (String family : columnFamily) {
            ColumnFamilyDescriptorBuilder descriptorBuilder =
                    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family));

            results.add(descriptorBuilder.setBlockCacheEnabled(true).build());
        }

        return results;
    }

    /**
     * 生成分区键
     * @param regionCount 分区数
     * @return 多个分区键
     */
    private byte[][] getSplitKeys(int regionCount) {
        int splitKeyCount = regionCount - 1;
        byte[][] bytes = new byte[splitKeyCount][];

        List<byte[]> byteList = new ArrayList<>();
        for (int i = 0; i < splitKeyCount; i++) {
            String key = i + "|";
            byteList.add(Bytes.toBytes(key));
        }

        byteList.toArray(bytes);
        return bytes;
    }


    public Connection getConnection() {
        return connection;
    }

    public Admin getAdmin() {
        try {
            admin = connection.getAdmin();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return admin;
    }


    /**
     * 用于创建线程池。
     * ThreadFactory实现类：重命名线程
     * @see ThreadFactory
     */
    private static class ThreadFactoryImpl implements ThreadFactory {
        private final String name;
        private AtomicInteger id = new AtomicInteger(1);

        private ThreadFactoryImpl(String name){
            this.name = "ThreadFactory-" + name + "-" + id.getAndIncrement();
        }

        @Override
        public Thread newThread(@Nonnull Runnable runnable) {
            return new Thread(runnable, name);
        }
    }
}

4、单元测试

package org.feng.hbase.util;

import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;

/**
 * Created by Feng on 2020/1/13 13:28
 * CurrentProject's name is hbase-demo
 * 测试
 */
public class HbaseUtilTest {

    private HbaseDao dao = null;

    @Before
    public void init(){
        dao = HbaseDao.getInstance();
    }

    @Test
    public void run() throws IOException {
        createTableTest();

        insertRecordTest();

        insertRecordsTest();

        scanByFilterTest();

        deleteColumnTest();

        deleteTableTest();
    }

    @Test
    public void testConnection(){
        Assert.assertNotNull(dao.getConnection());
    }

    @Test
    public void testAdmin() {
        Assert.assertNotNull(dao.getAdmin());
    }

    /**
     * 创建表空间
     */
    @Test
    public void testCreateNameSpace() throws IOException {
        // 不存在，创建
        dao.createNameSpace("feng_namespace_test");

        // 存在，不创建
        dao.createNameSpace("feng_namespace_test");

        Assert.assertNotNull(dao.getAdmin().getNamespaceDescriptor("feng_namespace_test"));
    }

    /**
     * 创建表；
     * 判断表是否存在；
     */
    private void createTableTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";
        dao.createTable(tableName, "username", "password");
        Assert.assertTrue(dao.tableExists(tableName));
    }

    /**
     * 插入数据；
     * 查询某一行指定列族，指定列的值
     */
    private void insertRecordTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";
        String columnFamily = "username";
        String rowKey1 = "1001";
        String rowKey = "1002";
        dao.insertRecord(tableName, rowKey1, columnFamily, "loginName", "小冯");
        dao.insertRecord(tableName, rowKey, columnFamily, "adminName", "admin");

        Assert.assertEquals("小冯", dao.selectValue(tableName, rowKey1, columnFamily, "loginName"));
        Assert.assertNotNull("admin", dao.selectValue(tableName, rowKey, columnFamily, "adminName"));
    }


    /**
     * 插入多个列；
     * 扫描全表
     */
    private void insertRecordsTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";
        String columnFamily = "username";

        String[] columns = {"father", "mother"};
        String[] values = {"fatherName", "montherName"};
        dao.insertRecords(tableName, "1001", columnFamily, columns, values);

        String result = dao.scanAllRecord(tableName);
        Assert.assertTrue(result.contains("fatherName"));
        Assert.assertTrue(result.contains("montherName"));
    }

    /**
     * 指定过滤器，扫描表
     */
    private void scanByFilterTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";

        // 列过滤器
        Filter filter = new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("father")));
        // 扫描到指定列
        dao.scanByFilter(tableName, filter)
                .forEach(cell -> {
                    Assert.assertTrue(new String(cell.getValueArray()).contains("fatherName"));
                    Assert.assertTrue(new String(cell.getValueArray()).contains("father"));
                });
    }

    /**
     * 删除列族；
     * 查询行；
     */
    private void deleteColumnTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";

        dao.deleteColumnFamily(tableName, "1001", "username");
        Assert.assertTrue(!dao.selectRow(tableName, "1001").contains("username"));
    }

    /**
     * 删除表；
     * 判断表是否存在
     */
    private void deleteTableTest() throws IOException {
        String tableName = "feng_namespace_test:feng123_user_test";
        dao.deleteTable(tableName);

        Assert.assertTrue(!dao.tableExists(tableName));
    }
}

5、另：ThreadLocal类的封装

只需要传入 Configuration 对象就可以操作。本人建议，封装其在 DAO 层，使用构造方法注入即可。若使用 SpringBoot 就更加方便了。

Configuration configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", "nat1");
configuration.set("hbase.zookeeper.property.clientPort", "2181");

package org.feng.data.hbase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;

import java.io.IOException;

/**
 * Created by Feng on 2019/11/21 10:19
 * @author Feng
 */
public abstract class AbstractHBaseSupport {

    private static ThreadLocal<Connection> connectionThreadLocal = new ThreadLocal<>();
    private static ThreadLocal<Admin> adminThreadLocal = new ThreadLocal<>();

    /**
     * 获取连接对象
     * @return Connection
     */
    protected synchronized Connection getConnection(Configuration configuration) {

        Connection connection = null;
        try {
            connection = connectionThreadLocal.get();
            if(connection == null){
                connection = ConnectionFactory.createConnection(configuration);
                connectionThreadLocal.set(connection);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return connection;
    }

    /**
     * 获取admin对象
     * @param configuration 配置信息
     * @return Admin
     */
    protected synchronized Admin getAdmin(Configuration configuration) {
        Admin admin = adminThreadLocal.get();
        if(admin == null){
            try {
                admin = getConnection(configuration).getAdmin();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return admin;
    }

    /**
     * 关闭资源：连接对象和admin对象
     * @throws IOException 关闭资源
     */
    protected void freeResource() throws IOException {
        Admin admin = adminThreadLocal.get();
        if(admin != null){
            admin.close();
            adminThreadLocal.remove();
        }

        Connection connection = connectionThreadLocal.get();
        if(connection != null){
            connection.close();
            connectionThreadLocal.remove();
        }
    }
}