golang-thrift 使用hbase教程
1.hbase知识
进入shell命令: /usr/local/hbase/bin/hbase shell
create 'ulive_test', 'i' // 创建一个列族 i
help // 查看所有命令
list // 查看所有表
describe 'student' // 查看表结构
// 删除表之前需要 disable
disable 'student'
drop 'student'
命名空间
list_namespace
创建空间
create_namespace 'gitxuzan'
create "gitxuzan:statue" ,'info'
disable "gitxuzan:statue"
drop "gitxuzan:stu"
drop_namespace 'gitxuzan'
put 'sut' ,'1001', 'info1:name', 'zhangsan' //新增数据1001是rowkey
get 'stu','1001' // 查询
更多命令相关使用查阅
2.Go使用Thrift连接HBase(resetful api接口效率要高,相关开源包:https://github.com/tsuna/gohbase)
mac 通过 brew安装，windows 系统下载 thrift.exe 可执行文件 (windows下载链接:http://archive.apache.org/dist/thrift/)
brew install thrift
查看版本 thrift -version (注意版本号最好是能在github.com/apache/thrift 找到对应tag)
Thrift version 0.14.1
3.下载HBase源码
https://hbase.apache.org/downloads.html
解压后来到 hbase-thrift\src\main\resources\org\apache\hadoop\hbase\thrift\Hbase.thrift
执行： thrift -gen go -o . ./Hbase.thrift
生成 gen-go 放到你的项目,注意里面可能有string 类型的语法错误,需要自己替换
4.go 源码部署
注意：thrift 插件和你使用的版本 https://github.com/apache/thrift 需要一致，我这边的代码是使用的thrift v0.14.1版本
一个链接和插入的例子(表要先创建)
package hbases
import (
"context"
"github.com/Xuzan9396/zlog"
"github.com/apache/thrift/lib/go/thrift"
"github.com/panjf2000/ants/v2"
"github.com/spf13/viper"
"myhbase/common"
"myhbase/global"
"myhbase/gen-go/hbase"
"net"
"strconv"
"sync"
"time"
)
type NewHbase struct {
data chan []*common.UserEventInfo
onces sync.Once
}
var HBASE *NewHbase
func GetModel() *NewHbase {
if HBASE == nil {
// 启动hbase
model := &NewHbase{
data: make(chan []*common.UserEventInfo, 2000),
}
HBASE = model
model.cient()
go model.HbasePull()
go model.ping_cient()
}
return HBASE
}
func (c *NewHbase) cient() {
host, portStr, table := viper.GetString("hbase.host"), viper.GetString("hbase.port"), viper.GetString("hbase.table")
var err error
// ------------------------- -----------------------
protocolFactory := thrift.NewTBinaryProtocolFactoryConf(&thrift.TConfiguration{
TBinaryStrictRead: thrift.BoolPtr(true),
TBinaryStrictWrite: thrift.BoolPtr(true),
})
zlog.F().Info(host," ",portStr)
global.Transport, err = thrift.NewTSocketConf(net.JoinHostPort(host, portStr),&thrift.TConfiguration{
ConnectTimeout: time.Second*60,
SocketTimeout: time.Second*60,
})
if err != nil {
zlog.F().Error(err)
return
}
client := hbase.NewHbaseClientFactory(global.Transport, protocolFactory)
if err := global.Transport.Open(); err != nil {
zlog.F().Error(err)
return
}
global.Hbase = client
argvalue0 := []byte(table)
value0 := hbase.Bytes(argvalue0)
bools, err := client.IsTableEnabled(context.TODO(),value0)
if err != nil {
zlog.F().Error(err)
}
if bools == true {
zlog.F().Info("表已存在:" + table)
}
}
func (c *NewHbase) ping_cient() {
defer func() {
if r := recover(); r != nil {
zlog.F().Error(r)
}
}()
times := time.NewTicker(5 * time.Second)
var err error
for {
select {
case <-times.C:
if global.Transport != nil && global.Transport.IsOpen() {
zlog.F().Info("已经isopen")
continue
}
zlog.F().Info("hbase断开链接,准备正在重连!")
//global.LOG.Info("test!")
host, portStr := viper.GetString("hbase.host"), viper.GetString("hbase.port")
// ------------------------- -----------------------
protocolFactory := thrift.NewTBinaryProtocolFactoryConf(&thrift.TConfiguration{
TBinaryStrictRead: thrift.BoolPtr(true),
TBinaryStrictWrite: thrift.BoolPtr(true),
})
zlog.F().Info(host," ",portStr)
global.Transport, err = thrift.NewTSocketConf(net.JoinHostPort(host, portStr),&thrift.TConfiguration{
ConnectTimeout: time.Second*60,
SocketTimeout: time.Second*60,
})
if err != nil {
zlog.F().Error(err)
return
}
client := hbase.NewHbaseClientFactory(global.Transport, protocolFactory)
if err := global.Transport.Open(); err != nil {
zlog.F().Error(err)
return
}
global.Hbase = client
zlog.F().Info("重新连接池状态正常")
}
}
}
func (c *NewHbase) HbasePull() {
defer func() {
if r := recover(); r != nil {
//global.LOG.Error("err", zap.Any("错误pull", r))
zlog.F().Error(r)
}
}()
p, _ := ants.NewPoolWithFunc(1, func(i interface{}) {
data, ok := i.([]*common.UserEventInfo)
if !ok {
return
}
c.Put(data)
})
defer p.Release()
for {
select {
case r := <-c.data:
_ = p.Invoke(r)
}
}
}
func (c *NewHbase) Set(data []*common.UserEventInfo) {
select {
case c.data <- data:
default:
//global.LOG.Info("丢弃了")
zlog.F().Info("丢弃了")
}
}
func (c *NewHbase) Put(data []*common.UserEventInfo) {
tableName, col := viper.GetString("hbase.table"), "i"
//col := "i"
sendMuData := make([]*hbase.BatchMutation, 0)
for _, value := range data {
if value.UserId <= 0 {
continue
}
userIdStr := strconv.FormatInt(value.UserId, 10)
timeStr := strconv.FormatInt(value.Time, 10)
if len(timeStr) == 13 {
timeStr = timeStr[:10]
}
widthStr := strconv.Itoa(value.Properties.ScreenWidth)
heightStr := strconv.Itoa(value.Properties.ScreenHeight)
rowKey := userIdStr + timeStr
//m := hbase.Mutation{false, []byte(col + ":name"), []byte("xuzan2"), true}
sendArr := []*hbase.Mutation{
{false, []byte(col + ":user_id"), []byte(userIdStr), true},
{false, []byte(col + ":event"), []byte(value.Event), true},
{false, []byte(col + ":device_id"), []byte(value.DeviceID), true},
{false, []byte(col + ":time"), []byte(timeStr), true},
{false, []byte(col + ":lib"), []byte(value.Properties.Lib), true},
{false, []byte(col + ":lib_version"), []byte(value.Properties.LibVersion), true},
{false, []byte(col + ":model"), []byte(value.Properties.Model), true},
{false, []byte(col + ":os"), []byte(value.Properties.Os), true},
{false, []byte(col + ":screen_width"), []byte(widthStr), true},
{false, []byte(col + ":screen_height"), []byte(heightStr), true},
{false, []byte(col + ":manufacturer"), []byte(value.Properties.Manufacturer), true},
{false, []byte(col + ":app_version"), []byte(value.Properties.AppVersion), true},
{false, []byte(col + ":element_id"), []byte(value.Properties.ElementID), true},
{false, []byte(col + ":activity"), []byte(value.Properties.Activity), true},
}
sendMuData = append(sendMuData, &hbase.BatchMutation{
[]byte(rowKey), sendArr,
})
}
if len(sendMuData) > 0 {
err := global.Hbase.MutateRows(
context.TODO(),
[]byte(tableName),
sendMuData,
nil)
if err == nil {
zlog.F().Info("批量插入成功")
} else {
zlog.F().Error(err)
}
}
}
5.遇到的bug
bug1: "err": "mutateRows failed: out of sequence response
看这个问题，处理方式是减少了并发请求了，之前是开了5个线程,一个连接去请求的，改成一个线程就好了,如果需要并发，可能需要多个连接 (相关原因：https://bbs.huaweicloud.com/blogs/197750)
bug2: : write: broken pip ，服务端修改了connect时间，服务器改了hbase配置hbase.thrift.server.socket.read.timeout (https://blog.csdn.net/u011342882/article/details/86300625)
6.其他相关链接
https://www.daryl.top/2021/04/go%E4%BD%BF%E7%94%A8thrift%E8%BF%9E%E6%8E%A5hbase/
https://blog.csdn.net/lesorb/article/details/64442673