Hbase存储小资源java代码

一 配置文件，根据实际做配置化管理

import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;

@ConfigurationProperties(prefix = "hbase")
@Getter
@Setter
public final class HBaseProperties {

    private String zkHost = "192.168.0.70";
    private String zkPort = "2181";
    private String clientMaxSize = "0";
    private String poolSize = "200";
    private String retryNum= "3";
    private String pause = "100";
    private String operationTimeout = "30000";
    private String remoteUsername = "root";
}

二 Hbase connection bean生成

import java.io.IOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.security.UserGroupInformation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
@EnableConfigurationProperties(HBaseProperties.class)
public class HBaseConfig {

    @Autowired
    HBaseProperties properties;

    private org.apache.hadoop.conf.Configuration buildHBaseConfiguration() {
        org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create();
        config.set("hbase.zookeeper.quorum", properties.getZkHost());
        config.set("hbase.zookeeper.property.clientPort", properties.getZkPort());
        config.set("hbase.client.keyvalue.maxsize", properties.getClientMaxSize());
        config.set("hbase.client.ipc.pool.size", properties.getPoolSize());
        config.set("hbase.client.retries.number", properties.getRetryNum());
        config.set("hbase.client.pause", properties.getPause());
        config.set("hbase.client.operation.timeout", properties.getOperationTimeout());
        return config;
    }

    @Bean(name = "hbaseConnection")
    Connection connection() {
        UserGroupInformation userGroupInformation = UserGroupInformation
            .createRemoteUser(properties.getRemoteUsername());
        try {
            return ConnectionFactory
                .createConnection(buildHBaseConfiguration(), User.create(userGroupInformation));
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
}

三 基础客户端

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

@Component
public class HBaseClient {

    @Autowired
    @Qualifier("hbaseConnection")
    Connection connection;

    private static final Logger LOG = LoggerFactory.getLogger(HBaseClient.class);

    /**
     * 判断表是否存在
     *
     * @param tableName 表名
     * @return true/false
     */
    public boolean isExists(String tableName) {
        boolean tableExists = false;
        try (Admin hbaseAdmin = connection.getAdmin()) {
            TableName table = TableName.valueOf(tableName);
            tableExists = hbaseAdmin.tableExists(table);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return tableExists;
    }

    /**
     * 创建表
     *
     * @param tableName    表名
     * @param columnFamily 列族
     * @return true/false
     */
    public boolean createTable(String tableName, List columnFamily) {
        return createTable(tableName, columnFamily, null);
    }

    /**
     * 预分区创建表
     *
     * @param tableName    表名
     * @param columnFamily 列族
     * @param keys         分区集合
     * @return true/false
     */
    public boolean createTable(String tableName, List columnFamily, List keys) {
        if (!isExists(tableName)) {
            try (Admin hbaseAdmin = connection.getAdmin()) {
                TableName table = TableName.valueOf(tableName);
                TableDescriptorBuilder descBuild = TableDescriptorBuilder.newBuilder(table);

                for (String cf : columnFamily) {
                    descBuild.setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf));
                }
                if (keys == null) {
                    hbaseAdmin.createTable(descBuild.build());
                } else {
                    byte[][] splitKeys = getSplitKeys(keys);
                    hbaseAdmin.createTable(descBuild.build(), splitKeys);
                }
                return true;
            } catch (IOException e) {
                e.printStackTrace();
            }
        } else {
            LOG.warn("{} already exists!!!", tableName);
            return false;
        }
        return false;
    }

    /**
     * 删除表
     *
     * @param tableName 表名
     */
    public void dropTable(String tableName) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            if (isExists(tableName)) {
                TableName table = TableName.valueOf(tableName);
                hbaseAdmin.disableTable(table);
                hbaseAdmin.deleteTable(table);
            }
        } catch (IOException e) {
            LOG.error("dropTable failed, msg = {}", e.getMessage(), e);
        }
    }

    /**
     * 插入数据（单条）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     * @param column       列
     * @param value        值
     * @return true/false
     */
    public boolean putData(String tableName, String rowKey, String columnFamily, String column,
        String value) {
        return putData(tableName, rowKey, columnFamily, Collections.singletonList(column),
            Collections.singletonList(value));
    }

    /**
     * 插入字节数组数据（单条）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     * @param column       列
     * @param value        值
     * @return true/false
     */
    public boolean putData(String tableName, String rowKey, String columnFamily, String column,
        byte[] value) {
        return putDataByByte(tableName, rowKey, columnFamily, Collections.singletonList(column),
            Collections.singletonList(value));
    }

    /**
     * 插入数据（批量）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     * @param columns      列
     * @param values       值
     * @return true/false
     */
    public boolean putData(String tableName, String rowKey, String columnFamily,
        List columns, List values) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Put put = new Put(Bytes.toBytes(rowKey));
            for (int i = 0; i < columns.size(); i++) {
                put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns.get(i)),
                    Bytes.toBytes(values.get(i)));
            }
            table.put(put);
            table.close();
            return true;
        } catch (IOException e) {
            LOG.error("putData failed, msg = {}", e.getMessage(), e);
            return false;
        }
    }

    /**
     * 插入字节数组数据（批量）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     * @param columns      列
     * @param values       值
     * @return true/false
     */
    public boolean putDataByByte(String tableName, String rowKey, String columnFamily,
        List columns, List values) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            this.createTable(tableName, Collections.singletonList(columnFamily));

            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Put put = new Put(Bytes.toBytes(rowKey));
            for (int i = 0; i < columns.size(); i++) {
                put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns.get(i)),
                    values.get(i));
            }
            table.put(put);
            table.close();
            return true;
        } catch (IOException e) {
            LOG.error("putData failed, msg = {}", e.getMessage(), e);
            return false;
        }
    }

    /**
     * 获取数据（全表数据）
     *
     * @param tableName 表名
     * @return map
     */
    public List> getData(String tableName) {
        List> list = new ArrayList<>();
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Scan scan = new Scan();
            ResultScanner resultScanner = table.getScanner(scan);
            for (Result result : resultScanner) {
                HashMap map = new HashMap<>();
                //rowkey
                String row = Bytes.toString(result.getRow());
                map.put("row", row);
                for (Cell cell : result.listCells()) {
                    //列族
                    String family = Bytes.toString(cell.getFamilyArray(),
                        cell.getFamilyOffset(), cell.getFamilyLength());
                    //列
                    String qualifier = Bytes.toString(cell.getQualifierArray(),
                        cell.getQualifierOffset(), cell.getQualifierLength());
                    //值
                    String data = Bytes.toString(cell.getValueArray(),
                        cell.getValueOffset(), cell.getValueLength());
                    map.put(family + ":" + qualifier, data);
                }
                list.add(map);
            }
            table.close();
        } catch (IOException e) {
            LOG.error("getData failed, msg = {}", e.getMessage(), e);
        }
        return list;
    }

    /**
     * 获取数据（根据rowkey）
     *
     * @param tableName 表名
     * @param rowKey    rowKey
     * @return map
     */
    public Map getData(String tableName, String rowKey) {
        HashMap map = new HashMap<>();
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(rowKey));
            Result result = table.get(get);
            if (result != null && !result.isEmpty()) {
                for (Cell cell : result.listCells()) {
                    //列族
                    String family = Bytes.toString(cell.getFamilyArray(),
                        cell.getFamilyOffset(), cell.getFamilyLength());
                    //列
                    String qualifier = Bytes.toString(cell.getQualifierArray(),
                        cell.getQualifierOffset(), cell.getQualifierLength());
                    //值
                    String data = Bytes.toString(cell.getValueArray(),
                        cell.getValueOffset(), cell.getValueLength());
                    map.put(family + ":" + qualifier, data);
                }
            }
            table.close();
        } catch (IOException e) {
            LOG.error("getData failed, msg = {}", e.getMessage(), e);
        }
        return map;
    }

    /**
     * 获取数据（根据rowkey，列族，列）
     *
     * @param tableName       表名
     * @param rowKey          rowKey
     * @param columnFamily    列族
     * @param columnQualifier 列
     * @return map
     */
    public String getData(String tableName, String rowKey, String columnFamily,
        String columnQualifier) {
        String data = "";
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(rowKey));
            get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier));
            Result result = table.get(get);
            if (result != null && !result.isEmpty()) {
                Cell cell = result.listCells().get(0);
                data = Bytes.toString(cell.getValueArray(), cell.getValueOffset(),
                    cell.getValueLength());
            }
            table.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return data;
    }

    /**
     * 获取数据（根据rowkey，列族，列）
     *
     * @param tableName       表名
     * @param rowKey          rowKey
     * @param columnFamily    列族
     * @param columnQualifier 列
     * @return map
     */
    public byte[] getByteData(String tableName, String rowKey, String columnFamily,
        String columnQualifier) {
        byte[] data = null;
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(rowKey));
            get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier));
            Result result = table.get(get);
            if (result != null && !result.isEmpty()) {
                Cell cell = result.listCells().get(0);
                data = Bytes.copy(cell.getValueArray(), cell.getValueOffset(),
                    cell.getValueLength());
            }
            table.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return data;
    }

    /**
     * 删除数据（根据rowkey）
     *
     * @param tableName 表名
     * @param rowKey    rowKey
     */
    public void deleteData(String tableName, String rowKey) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Delete delete = new Delete(Bytes.toBytes(rowKey));
            table.delete(delete);
            table.close();
        } catch (IOException e) {
            LOG.error("deleteData failed, msg = {}", e.getMessage(), e);
        }
    }

    /**
     * 删除数据（根据rowkey，列族）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     */
    public void deleteData(String tableName, String rowKey, String columnFamily) {
        try (Admin hbaseAdmin = connection.getAdmin()) {

            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Delete delete = new Delete(Bytes.toBytes(rowKey));
            delete.addFamily(columnFamily.getBytes());
            table.delete(delete);
            table.close();
        } catch (IOException e) {
            LOG.error("deleteData failed, msg = {}", e.getMessage(), e);
        }
    }

    /**
     * 删除数据（根据rowkey，列族）
     *
     * @param tableName    表名
     * @param rowKey       rowKey
     * @param columnFamily 列族
     * @param column       列
     */
    public void deleteData(String tableName, String rowKey, String columnFamily, String column) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Delete delete = new Delete(Bytes.toBytes(rowKey));
            delete.addColumn(columnFamily.getBytes(), column.getBytes());
            table.delete(delete);
            table.close();
        } catch (IOException e) {
            LOG.error("deleteData failed, msg = {}", e.getMessage(), e);
        }
    }

    /**
     * 删除数据（多行）
     *
     * @param tableName 表名
     * @param rowKeys   rowKey集合
     */
    public void deleteData(String tableName, List rowKeys) {
        try (Admin hbaseAdmin = connection.getAdmin()) {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            List deleteList = new ArrayList<>();
            for (String row : rowKeys) {
                Delete delete = new Delete(Bytes.toBytes(row));
                deleteList.add(delete);
            }
            table.delete(deleteList);
            table.close();
        } catch (IOException e) {
            LOG.error("deleteData failed, msg = {}", e.getMessage(), e);
        }
    }

    /**
     * 分区【10, 20, 30】 -> ( ,10] (10,20] (20,30] (30, )
     *
     * @param keys 分区集合[10, 20, 30]
     * @return byte二维数组
     */
    private byte[][] getSplitKeys(List keys) {
        byte[][] splitKeys = new byte[keys.size()][];
        TreeSet rows = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        for (String key : keys) {
            rows.add(Bytes.toBytes(key));
        }
        int i = 0;
        for (byte[] row : rows) {
            splitKeys[i] = row;
            i++;
        }
        return splitKeys;
    }
}

四 测试类

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

@SpringBootTest
class HBaseClientTest {

    private static final Logger LOG = LoggerFactory.getLogger(HBaseClientTest.class);
    @Autowired
    HBaseClient hBaseClient;

    @Test
    void isExists() {
        String tn = "song20230815";
        hBaseClient.getByteData(tn, "row1", "cf", "a");
        hBaseClient.putData(tn, "row1", "cf", "a", "file.getBytes()");

        System.out.println("---开始创建test表---");
        hBaseClient.createTable(tn, Collections.singletonList("cf"));

        System.out.println("---判断test表是否存在---");
        Boolean t = hBaseClient.isExists(tn);
        System.out.println(t);

        System.out.println("\n---插入一列数据---");
        hBaseClient.putData(tn, "row1", "cf", "a", "value1-1");

        System.out.println("\n---插入多列数据---");
        hBaseClient.putData(tn, "row2", "cf",
            Arrays.asList("a", "b", "c"), Arrays.asList("value2-1", "value2-2", "value2-3"));

        System.out.println("\n---根据rowkey、列族、列查询数据---");
        String columnData = hBaseClient.getData(tn, "row2", "cf", "b");
        System.out.println(columnData);

        System.out.println("\n---根据rowkey查询数据---");
        Map rowData = hBaseClient.getData(tn, "row2");
        System.out.println(rowData);

        System.out.println("\n---查询全表数据---");
        List> tableData = hBaseClient.getData("tn");
        System.out.println(tableData);

        System.out.println("\n---根据rowkey、列族、列删除数据---");
        hBaseClient.deleteData(tn, "row2", "cf", "b");

        System.out.println("\n---根据rowkey、列族删除数据---");
        hBaseClient.deleteData(tn, "row2", "cf");

        System.out.println("\n---根据rowkey删除数据---");
        hBaseClient.deleteData(tn, "row2");

        System.out.println("\n---根据rowkey批量删除数据---");
        hBaseClient.deleteData(tn, Arrays.asList("row1", "row2"));

        System.out.println("\n---删除表---");
        hBaseClient.dropTable(tn);
    }
}