最近项目使用到hbase当做数据库存储数据,经过多番网上搜索查找对hbase操作的功能写成了通用方法,希望对大家有所帮助,后期有新的通用功能会实时更新的,通用方法不采用springboot集成的hbase,使用的事原生hbase方法。
开发环境
系统:windows
JDK:1.8
开发工具:Intellij IDEA
虚拟机:vmware
三台linux节点: 172.162.15.111、172.162.15.112、172.162.15.113
Linux:CentOS 7 迷你版 CentOS-7-x86_64-Minimal-1708.iso
Hadoop: Hadoop-3.1.2
Hbase : hbase-2.0.5
Zookeeper: zookeeper-3.4.1
第二步:配置windows系统host,根据您的需求,这一步可有可无
#虚拟机访问地址
172.162.15.111 davidone
172.162.15.112 davidtwo
172.162.15.113 davidthree
第三步:项目结构配置pom.xml
org.springframework.boot
spring-boot-configuration-processor
true
org.apache.hbase
hbase-client
2.0.5
org.slf4j
slf4j-log4j12
log4j
log4j
javax.servlet
servlet-api
application.yml 这个配置以增量的方式添加到文件
logging:
config: classpath:Logback.xml
bigdata:
config:
hBaseConfigMap: {hbase.zookeeper.quorum : 'davidone:2181,davidtwo:2181,davidthree:2181'}
日志配置文件:Logback.xml
${PATTERN}
UTF-8
${OPEN_FILE_PATH}/debug/topic_file_debug.%d{yyyy-MM-dd}-%i.log
180
100MB
true
%d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
utf-8
DEBUG
ACCEPT
DENY
${OPEN_FILE_PATH}/info/topic_file_sync.%d{yyyy-MM-dd}-%i.log
180
100MB
true
%d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
utf-8
INFO
ACCEPT
DENY
${OPEN_FILE_PATH}/error/topic_file_error.%d{yyyy-MM-dd}-%i.log
180
100MB
true
%d{yyyy-MM-dd HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} $ %msg%n
utf-8
ERROR
ACCEPT
DENY
第四步:核心代码
BigDataConfig.java 配置文件信息
@Component()
@ConfigurationProperties(prefix = "bigdata.config")
public class BigDataConfig {
private Map hBaseConfigMap;
public Map gethBaseConfigMap() {
return hBaseConfigMap;
}
public void sethBaseConfigMap(Map hBaseConfigMap) {
this.hBaseConfigMap = hBaseConfigMap;
}
}
HBaseRepository.java 通用操作hbase数据代码,包含了通用的二级搜索
@Component
public class HBaseRepository{
private Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
private BigDataConfig bigDataConfig;
private static Configuration configuration = HBaseConfiguration.create();
//设置连接池
private static ExecutorService executorServicePoolSize = Executors.newScheduledThreadPool(20);
private ThreadLocal connectionThreadLocal = new ThreadLocal<>();
private ThreadLocal adminThreadLocal = new ThreadLocal<>();
private static Connection connection = null;
private static Admin admin = null;
private final byte[] POSTFIX = new byte[] { 0x00 };
/**
* 加载HBase配置信息
*/
private void initConfigurationInfo(){
Map hBaseConfigMap = bigDataConfig.gethBaseConfigMap();
if(hBaseConfigMap.size() == 0){
logger.debug(MessageFormat.format("HBase配置信息初始化失败:{0}",JSON.toJSONString(hBaseConfigMap)));
}else{
for (Map.Entry confEntry : hBaseConfigMap.entrySet()) {
configuration.set(confEntry.getKey(), confEntry.getValue());
}
}
}
/**
* 初始化HBase client admin
*
*/
private void initHBaseClientAdmin(){
try{
admin = adminThreadLocal.get();
if(admin == null && connection != null){
admin = connection.getAdmin();
adminThreadLocal.set(admin);
}else{
logger.debug(MessageFormat.format("创建hBase connection连接 失败:{0}",connection));
}
}catch (Exception e){
logger.error(MessageFormat.format("初始化hBase client admin 客户端管理失败:错误信息:{0}",e));
}
}
/**
* 初始化HBase资源
*/
public void initRepository(){
try{
initConfigurationInfo();
connection = connectionThreadLocal.get();
if(connection == null){
connection = ConnectionFactory.createConnection(configuration, executorServicePoolSize);
connectionThreadLocal.set(connection);
}
initHBaseClientAdmin();
}catch (Exception e){
logger.error(MessageFormat.format("创建hBase connection 连接失败:{0}",e));
e.printStackTrace();
}finally {
close(admin,null,null);
}
}
/**
* 获取table
* @param tableName 表名
* @return Table
*/
private Table getTable(String tableName) {
try {
return connection.getTable(TableName.valueOf(tableName));
}catch (Exception e){
e.printStackTrace();
return null;
}
}
/**
* 同时创建多张数据表
* @param tableMap 数据表 map<表名,列簇集合>
* @return
*/
public boolean createManyTable(Map> tableMap){
try{
if(admin != null){
for (Map.Entry> confEntry : tableMap.entrySet()) {
createTable(confEntry.getKey(), confEntry.getValue());
}
}
}catch (Exception e){
logger.error(MessageFormat.format("创建多个表出现未知错误:{0}",e.getMessage()));
e.printStackTrace();
return false;
}finally {
close(admin,null,null);
}
return true;
}
/**
* 创建hbase表和列簇
* @param tableName 表名
* @param columnFamily 列簇
* @return 1:创建成功;0:创建出错;2:创建的表存在
*/
public int createOneTable (String tableName,String... columnFamily){
try{
//创建表,先查看表是否存在,然后在删除重新创建
if(admin != null){
return createTable(tableName,Arrays.asList(columnFamily));
}else{
logger.error("admin变量没有初始化成功");
return 0;
}
}catch (Exception e){
logger.debug(MessageFormat.format("创建表失败:{0},错误信息是:{1}",tableName,e.getMessage()));
e.printStackTrace();
return 0;
}finally {
close(admin,null,null);
}
}
/**
*
* @param tableName
* @param columnFamily
* @return
* @throws Exception
*/
private int createTable(String tableName,List columnFamily) throws Exception {
if(admin.tableExists(TableName.valueOf(tableName))){
logger.debug(MessageFormat.format("创建HBase表名:{0} 在HBase数据库中已经存在",tableName));
return 2;
}else{
List familyDescriptors =new ArrayList<>(columnFamily.size());
for(String column : columnFamily){
familyDescriptors.add(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(column)).build());
}
TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
.setColumnFamilies(familyDescriptors).build();
admin.createTable(tableDescriptor);
logger.info(MessageFormat.format("创建表成功:表名:{0},列簇:{1}",tableName,JSON.toJSONString(columnFamily)));
return 1;
}
}
/**
* 插入or 更新记录(单行单列族-多列多值)
* @param tableName 表名
* @param row 行号 唯一
* @param columnFamily 列簇名称
* @param columns 多个列
* @param values 对应多个列的值
*/
public boolean insertManyColumnRecords(String tableName,String row,String columnFamily,List columns,List values){
try{
Table table = getTable(tableName);
Put put = new Put(Bytes.toBytes(row));
for(int i = 0; i < columns.size(); i++){
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(columns.get(i)),Bytes.toBytes(values.get(i)));
table.put(put);
}
logger.info(MessageFormat.format("添加单行单列族-多列多值数据成功:表名:{0},列名:{1},列值:{2}",tableName, JSON.toJSONString(columns),JSON.toJSONString(values)));
return true;
}catch (Exception e){
e.printStackTrace();
logger.debug(MessageFormat.format("添加单行单列族-多列多值数据失败:表名:{0};错误信息是:{1}",tableName,e.getMessage()));
return false;
}
}
/**
* 插入or更新记录(单行单列族-单列单值)
* @param tableName 表名
* @param row 行号 唯一
* @param columnFamily 列簇名称
* @param column 列名
* @param value 列的值
*/
public boolean insertOneColumnRecords(String tableName,String row,String columnFamily,String column,String value){
try{
Table table = getTable(tableName);
Put put = new Put(Bytes.toBytes(row));
put.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(column),Bytes.toBytes(value));
table.put(put);
logger.info(MessageFormat.format("添加单行单列族-单列单值数据成功:表名:{0},列名:{1},列值:{2}",tableName,column,value));
return true;
}catch (Exception e){
logger.debug(MessageFormat.format("添加单行单列族-单列单值数据失败:表名:{0},错误信息是:{1}",tableName,e.getMessage()));
e.printStackTrace();
return false;
}
}
/**
* 根据行号删除表中一条记录
* @param tableName 表名
* @param rowNumber 行号
*/
public boolean deleteDataByRowNumber(String tableName,String rowNumber){
try{
Table table = getTable(tableName);
Delete delete = new Delete(Bytes.toBytes(rowNumber));
table.delete(delete);
logger.info(MessageFormat.format("根据行号删除表中记录成功:表名:{0},行号:{1}",tableName,rowNumber));
return true;
}catch (Exception e){
e.printStackTrace();
logger.debug(MessageFormat.format("根据行号删除表中记录失败:表名:{0},行号:{1}",tableName,rowNumber));
return false;
}
}
/**
* 删除列簇下所有数据
* @param tableName 表名
* @param columnFamily 列簇
*/
public boolean deleteDataByColumnFamily(String tableName,String columnFamily){
try{
if(!admin.tableExists(TableName.valueOf(tableName))){
logger.debug(MessageFormat.format("根据行号和列簇名称删除这行列簇相关的数据失败:表名不存在:{0}",tableName));
return false;
}
admin.deleteColumnFamily(TableName.valueOf(tableName),Bytes.toBytes(columnFamily));
logger.info(MessageFormat.format("删除该表中列簇下所有数据成功:表名:{0},列簇:{1}",tableName,columnFamily));
return true;
}catch (Exception e){
e.printStackTrace();
logger.debug(MessageFormat.format("删除该表中列簇下所有数据失败:表名:{0},列簇:{1},错误信息:{2}",tableName,columnFamily,e.getMessage()));
return false;
}
}
/**
* 删除指定的列 ->删除最新列,保留旧列。
* 如 相同的rowkey的name列数据 提交两次数据,此方法只会删除最近的数据,保留旧数据
* @param tableName 表名
* @param rowNumber 行号
* @param columnFamily 列簇
* @param cloumn 列
*/
public boolean deleteDataByColumn(String tableName,String rowNumber,String columnFamily,String cloumn){
try{
if(!admin.tableExists(TableName.valueOf(tableName))){
logger.debug(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列失败:表名不存在:{0}",tableName));
return false;
}
Table table = getTable(tableName);
Delete delete = new Delete(rowNumber.getBytes());
delete.addColumn(Bytes.toBytes(columnFamily),Bytes.toBytes(cloumn));
table.delete(delete);
logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列成功:表名:{0},行号:{1},列簇:{2},列:{3}",tableName,rowNumber,columnFamily,cloumn));
return true;
}catch (Exception e){
e.printStackTrace();
logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->删除最新列,保留旧列失败:表名:{0},行号:{1},列簇:{2},列:{3},错误信息:{4}",tableName,rowNumber,columnFamily,cloumn,e.getMessage()));
return false;
}
}
/**
* 删除指定的列 ->新旧列都会删除
* @param tableName 表名
* @param rowNumber 行号
* @param columnFamily 列簇
* @param cloumn 列
*/
public boolean deleteDataByAllcolumn(String tableName,String rowNumber,String columnFamily,String cloumn){
try{
if(!admin.tableExists(TableName.valueOf(tableName))){
logger.debug(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除失败:表名不存在:{0}",tableName));
return false;
}
Table table = getTable(tableName);
Delete delete = new Delete(rowNumber.getBytes());
delete.addColumns(Bytes.toBytes(columnFamily),Bytes.toBytes(cloumn));
table.delete(delete);
logger.info(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除成功:表名:{0},行号:{1},列簇:{2},列:{3}",tableName,rowNumber,columnFamily,cloumn));
return true;
}catch (Exception e){
e.printStackTrace();
logger.error(MessageFormat.format("根据行号表名列簇删除指定列 ->新旧列都会删除失败:表名:{0},行号:{1},列簇:{2},列:{3},错误信息:{4}",tableName,rowNumber,columnFamily,cloumn,e.getMessage()));
return false;
}
}
/**
* 删除表
* @param tableName 表名
*/
public boolean deleteTable(String tableName){
try{
TableName table = TableName.valueOf(tableName);
if(admin.tableExists(table)){
//禁止使用表,然后删除表
admin.disableTable(table);
admin.deleteTable(table);
}
logger.info(MessageFormat.format("删除表成功:{0}",tableName));
return true;
}catch (Exception e){
e.printStackTrace();
logger.debug(MessageFormat.format("删除表失败:{0},错误信息是:{1}",tableName,e.getMessage()));
return false;
}finally {
close(admin,null,null);
}
}
/**
* 查询所有表名
*/
public List getAllTableNames(){
List resultList = new ArrayList<>();
try {
TableName[] tableNames = admin.listTableNames();
for(TableName tableName : tableNames){
resultList.add(tableName.getNameAsString());
}
logger.info(MessageFormat.format("查询库中所有表的表名成功:{0}",JSON.toJSONString(resultList)));
}catch (IOException e) {
logger.error("获取所有表的表名失败",e);
}finally {
close(admin,null,null);
}
return resultList;
}
/**
* 根据表名和行号查询数据
* @param tableName 表名
* @param rowNumber 行号
* @return
*/
public Map selectOneRowDataMap(String tableName,String rowNumber){
Map resultMap = new HashMap<>();
Get get = new Get(Bytes.toBytes(rowNumber));
Table table = getTable(tableName);
try{
Result result = table.get(get);
if(result !=null && !result.isEmpty()){
for(Cell cell : result.listCells()){
resultMap.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())
);
}
}
logger.info(MessageFormat.format("根据表名和行号查询数据:表名:{0},行号:{1},查询结果:{2}",tableName,rowNumber,JSON.toJSONString(resultMap)));
}catch (Exception e){
e.printStackTrace();
logger.debug(MessageFormat.format("根据表名和行号查询数据失败:表名:{0},行号:{1},错误信息:{2}",tableName,rowNumber,e.getMessage()));
}finally {
close(null,null,table);
}
return resultMap;
}
/**
* 根据不同条件查询数据
* @param tableName 表名
* @param columnFamily 列簇
* @param queryParam 过滤列集合 ("topicFileId,6282")=>("列,值")
* @param regex 分隔字符
* @param bool 查询方式:and 或 or | true : and ;false:or
*
* @return
*/
public List
测试HBaseRepository功能
@RunWith(SpringRunner.class)
@SpringBootTest
public class HBaseUtilsTest {
@Autowired
private HBaseRepository hBaseRepository;
@Test
public void initTable(){
hBaseRepository.initRepository();
Map> tableMap = new HashMap<>();
tableMap.put("david_topic:actionFlow",Arrays.asList("info","logs"));
tableMap.put("david_topic:recording",Arrays.asList("info","logs"));
tableMap.put("david_topic:syncLog",Arrays.asList("info","logs"));
tableMap.put("david_topic:uploadVersion",Arrays.asList("info","logs"));
hBaseRepository.createManyTable(tableMap);
}
@Test
public void createOneTable(){
//在指定命名空间创建表
hBaseRepository.createOneTable("david_topic:topictest","info","log");
}
@Test
public void deleteTable(){
hBaseRepository.deleteTable("david_topic:topictest");
}
@Test
public void insertManyColumnRecords(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
String columnFamily = "info";
List columns = Arrays.asList("id", "title", "poform", "dept", "level", "createUser");
List values = Arrays.asList("105155", "瑞风S4品牌临时页面", "Mobile", "营销中心", "普通", "王丽");
hBaseRepository.insertManyColumnRecords(tableName, rowNumber, columnFamily, columns, values);
}
@Test
public void selectTableByRowNumberAndColumnFamily(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
String columnFamily = "info";
hBaseRepository.selectTableByRowNumberAndColumnFamily(tableName,rowNumber,columnFamily);
}
@Test
public void getAllTableNames(){
hBaseRepository.getAllTableNames();
}
@Test
public void insertOneColumnRecords(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
String columnFamily = "info";
hBaseRepository.insertOneColumnRecords(tableName, rowNumber, columnFamily, "delete", "否");
}
@Test
public void deleteDataBycolumn(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
String columnFamily = "info";
String column = "delete";
hBaseRepository.deleteDataByColumn(tableName,rowNumber,columnFamily,column);
}
@Test
public void getTableDataCount(){
String tableName = "david_topic:topictest";
hBaseRepository.getTableDataCount(tableName,"log");
}
@Test
public void deleteDataByRowNumber(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
hBaseRepository.deleteDataByRowNumber(tableName, rowNumber);
}
@Test
public void selectTableAllDataMap(){
String tableName = "david_topic:topictest";
hBaseRepository.selectTableAllDataMap(tableName);
}
@Test
public void selectTableAllDataMapColumnFamily(){
String tableName = "david_topic:topictest";
String columnFamily = "log";
hBaseRepository.selectTableAllDataMap(tableName,columnFamily);
}
@Test
public void selectOneRowDataMap(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
hBaseRepository.selectOneRowDataMap(tableName,rowNumber);
}
@Test
public void selectColumnValue(){
String tableName = "david_topic:topictest";
String rowNumber = "105155";
String columnFamily = "info";
String column = "title";
hBaseRepository.selectColumnValue(tableName,rowNumber,columnFamily,column);
}
@Test
public void deleteDataByRowNumberAndColumnFamily(){
String tableName = "david_topic:topictest";
String columnFamily = "info";
hBaseRepository.deleteDataByColumnFamily(tableName,columnFamily);
}
@Test
public void selectTableDataByFilter(){
String tableName = "david_topic:topictest";
List queryParam = Arrays.asList("id,105155","dept,营销中心");
hBaseRepository.selectTableDataByFilter(tableName, "info", queryParam, ",",true);
}
@Test
public void selectColumnValueDataByFilter(){
String tableName = "david_topic:topictest";
List queryParam = Arrays.asList("topicFileId,6282");
hBaseRepository.selectColumnValueDataByFilter(tableName, "info", queryParam, ",","id",true);
}
/**
* 分页方式一:
* 全表扫描并返回经过分页后的数据
*/
@Test
public void selectTableAllDataMapAllPage(){
String tableName ="david_topic:topictest";
int pageSize = 3;
String key = null;
int dataCount = pageSize;
while (dataCount == pageSize) {
String mobileKey = null;
if (key != null) {
mobileKey = key;
}
List> result = hBaseRepository.selectTableAllDataMapPage(tableName, pageSize, mobileKey);
if(result != null && result.size() > 0 ){
dataCount = result.size();
key = result.get(dataCount-1).get("rowKey").toString();
System.out.println(key);
}else{
break;
}
}
}
/**
* 分页方式二:
* 返回每页数据,下一页需要入参【上一页最后一位的rowKey】
*/
@Test
public void selectTableAllDataMapPage(){
String tableName ="david_topic:topictest";
int pageSize = 3;
List> result = hBaseRepository.selectTableAllDataMapPage(tableName, pageSize, "32956703305764864");
System.out.println(result);
}
}