parquet--golang使用
github
其实如果不适用一些可视化工具解析parquet文件，不太好看parquet文件内部正常应该是什么样的。但是使用一些可视化工具的话，可以发现，parquet文件会像表格，如excel文件，csv文件那样，排列数据。通过结构体写入的时候也和写入csv文件很像，写一次就是写一行。
安装
go get github.com/xitongsys/parquet-go
type Student struct {
Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
Age int32 `parquet:"name=age, type=INT32, encoding=PLAIN"`
Id int64 `parquet:"name=id, type=INT64"`
Weight float32 `parquet:"name=weight, type=FLOAT"`
Sex bool `parquet:"name=sex, type=BOOLEAN"`
Day int32 `parquet:"name=day, type=INT32, convertedtype=DATE"`
Ignored int32 //without parquet tag and won't write
}
这里的结构体中，如果有切片类型的属性，就要使用valuetype标签，因为源码中会判断属性的类型
type Acc struct {
Accel []float32 `parquet:"name=accel, valuetype=FLOAT" json:"accel,omitempty"`
Gyro []float32 `parquet:"name=gyro, valuetype=FLOAT" json:"gyro,omitempty"`
Comp []float32 `parquet:"name=comp, valuetype=FLOAT" json:"comp,omitempty"`
}
func TestParquetExp(t *testing.T) {
var err error
w, err := os.Create("./flat.parquet")
if err != nil {
log.Println("Can't create local file", err)
return
}
//write
pw, err := writer.NewParquetWriterFromWriter(w, new(Student), 4)//使用4个协程，并发写
if err != nil {
log.Println("Can't create parquet writer", err)
return
}
pw.RowGroupSize = 128 * 1024 * 1024 //128M
pw.CompressionType = parquet.CompressionCodec_SNAPPY
num := 100
for i := 0; i < num; i++ {
stu := Student{
Name: "StudentName",
Age: int32(20 + i%5),
Id: int64(i),
Weight: float32(50.0 + float32(i)*0.1),
Sex: bool(i%2 == 0),
Day: int32(time.Now().Unix() / 3600 / 24),
}
if err = pw.Write(stu); err != nil { //写了100次，也就是100行
log.Println("Write error", err)
}
}
if err = pw.WriteStop(); err != nil {
log.Println("WriteStop error", err)
return
}
log.Println("Write Finished")
w.Close()
///read
fr, err := local.NewLocalFileReader("./flat.parquet")
if err != nil {
log.Println("Can't open file")
return
}
pr, err := reader.NewParquetReader(fr, new(Student), 4)
if err != nil {
log.Println("Can't create parquet reader", err)
return
}
num = int(pr.GetNumRows()) //写了100行，这里就返回100
for i := 0; i < num; i++ {
//if i%2 == 0 {
// pr.SkipRows(10) //skip 10 rows 0-9,20-29,40-49,60-69,80-89被跳过
// continue
//}
stus := make([]Student, 2) //这里的切片长度代表一次都多少行
if err = pr.Read(&stus); err != nil {
log.Println("Read error", err)
}
log.Println(stus)
}
pr.ReadStop()
fr.Close()
}
解析parquet文件
解析parquet文件
源码
func NewSchemaHandlerFromStruct(obj interface{}) (sh *SchemaHandler, err error）
func NewSchemaHandlerFromStruct(obj interface{}) (sh *SchemaHandler, err error) {
defer func() {
if r := recover(); r != nil {
switch x := r.(type) {
case string:
err = errors.New(x)
case error:
err = x
default:
err = errors.New("error occurred")
}
}
}()
ot := reflect.TypeOf(obj).Elem() //获取传入的obj的类型（结构体？切片？）
item := NewItem() //该方法中主要使用的实例
item.GoType = GoType //给item的info属性的属性赋值
item.Info.InName = "Parquet_go_root" //给item的info属性的属性赋值
item.Info.ExName = "parquet_go_root" //给item的info属性的属性赋值
item.Info.RepetitionType = parquet.FieldRepetitionType_REQUIRED //给item的info属性的属性赋值
stack := make([]*Item, 1) //该数组应该是存放obj有几个属性
stack[0] = NewItem //一开始，item是空的，stack也只有一个item
schemaElements := make([]*parquet.SchemaElement, 0) //存放模板元素的切片
infos := make([]*common.Tag, 0) //应该是每一个属性的信息
for len(stack) > 0 { //stack刚开始长度为1时，主要是在准备将一些东西放进去，如果是obj是结构体，就是要把obj的属性封装为*Item类型的实例放进去，用来处理。之后才是真正处理stack里面的内容
ln := len(stack)
item = stack[ln-1]
stack = stack[:ln-1] //1.对于刚开始进入该循环的时候，这里的作用是将stack置空，用来之后存放封装了obj的属性的item 2.之后是真正的便开始处理每个obj的属性
var newInfo *common.Tag
if item.GoType.Kind() == reflect.Struct { //obj如果是结构体类型
schema := parquet.NewSchemaElement() //表示模板中的元素。
schema.Name = item.Info.InName
schema.RepetitionType = &item.Info.RepetitionType
numField := int32(item.GoType.NumField()) //obj中的属性个数
schema.NumChildren = &numField
schemaElements = append(schemaElements, schema) //放一个模板元素进去
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo) //item这个时候其实只有类型（GoType里面的东西）与传入的obj相关，并且复制了一个新的info
infos = append(infos, newInfo) //把新的info存上
for i := int(numField - 1); i >= 0; i-- { //开始遍历obj的属性
f := item.GoType.Field(i) //获取obj的属性
tagStr := f.Tag.Get("parquet") //获取属性的parquet 标签
//ignore item without parquet tag
if len(tagStr) <= 0 { //没有qarquet标签就跳过，忽略
numField--
continue
}
newItem := NewItem()
//把字符标签“name=comp, type=FLOAT”，转换为*Tag类型的实例(以“,”拆分，再以“=”拆分)InName = {string} "Comp" ExName = {string} "comp" Type = {string} "FLOAT"
newItem.Info, err = common.StringToTag(tagStr)
if err != nil {
return nil, fmt.Errorf("failed parse tag: %s", err.Error())
}
newItem.Info.InName = f.Name //将使用反射从结构获取的属性名称赋值给这个item的info属性的InName属性，但是这个属性已经有正确的值了呀
newItem.GoType = f.Type //将使用反射从结构获取的属性类型赋值给item的GoType属性
if f.Type.Kind() == reflect.Ptr {
newItem.GoType = f.Type.Elem()
newItem.Info.RepetitionType = parquet.FieldRepetitionType_OPTIONAL
}
stack = append(stack, newItem) //将封装了obj一个属性的item存起来
}
} else if item.GoType.Kind() == reflect.Slice &&
item.Info.RepetitionType != parquet.FieldRepetitionType_REPEATED {
schema := parquet.NewSchemaElement()
schema.Name = item.Info.InName
rt1 := item.Info.RepetitionType
schema.RepetitionType = &rt1
var numField int32 = 1
schema.NumChildren = &numField
ct1 := parquet.ConvertedType_LIST
schema.ConvertedType = &ct1
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo)
infos = append(infos, newInfo)
schema = parquet.NewSchemaElement()
schema.Name = "List"
rt2 := parquet.FieldRepetitionType_REPEATED
schema.RepetitionType = &rt2
schema.NumChildren = &numField
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo)
newInfo.InName, newInfo.ExName = "List", "list"
infos = append(infos, newInfo)
newItem := NewItem()
newItem.Info = common.GetValueTagMap(item.Info) //有问题
newItem.Info.InName = "Element"
newItem.Info.ExName = "element"
newItem.GoType = item.GoType.Elem()
if newItem.GoType.Kind() == reflect.Ptr {
newItem.Info.RepetitionType = parquet.FieldRepetitionType_OPTIONAL
newItem.GoType = item.GoType.Elem().Elem()
} else {
newItem.Info.RepetitionType = parquet.FieldRepetitionType_REQUIRED
}
stack = append(stack, newItem)
} else if item.GoType.Kind() == reflect.Slice &&
item.Info.RepetitionType == parquet.FieldRepetitionType_REPEATED {
newItem := NewItem()
newItem.Info = item.Info
newItem.GoType = item.GoType.Elem()
stack = append(stack, newItem)
} else if item.GoType.Kind() == reflect.Map {
schema := parquet.NewSchemaElement()
schema.Name = item.Info.InName
rt1 := item.Info.RepetitionType
schema.RepetitionType = &rt1
var numField1 int32 = 1
schema.NumChildren = &numField1
ct1 := parquet.ConvertedType_MAP
schema.ConvertedType = &ct1
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo)
infos = append(infos, newInfo)
schema = parquet.NewSchemaElement()
schema.Name = "Key_value"
rt2 := parquet.FieldRepetitionType_REPEATED
schema.RepetitionType = &rt2
var numField2 int32 = 2
schema.NumChildren = &numField2
ct2 := parquet.ConvertedType_MAP_KEY_VALUE
schema.ConvertedType = &ct2
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo)
newInfo.InName, newInfo.ExName = "Key_value", "key_value"
infos = append(infos, newInfo)
newItem := NewItem()
newItem.Info = common.GetValueTagMap(item.Info)
newItem.GoType = item.GoType.Elem()
if newItem.GoType.Kind() == reflect.Ptr {
newItem.Info.RepetitionType = parquet.FieldRepetitionType_OPTIONAL
newItem.GoType = item.GoType.Elem().Elem()
} else {
newItem.Info.RepetitionType = parquet.FieldRepetitionType_REQUIRED
}
stack = append(stack, newItem)
newItem = NewItem()
newItem.Info = common.GetKeyTagMap(item.Info)
newItem.GoType = item.GoType.Key()
newItem.Info.RepetitionType = parquet.FieldRepetitionType_REQUIRED
stack = append(stack, newItem)
} else {
schema, err := common.NewSchemaElementFromTagMap(item.Info)
if err != nil {
return nil, fmt.Errorf("failed to create schema from tag map: %s", err.Error())
}
schemaElements = append(schemaElements, schema)
newInfo = common.NewTag()
common.DeepCopy(item.Info, newInfo)
infos = append(infos, newInfo)
}
}
res := NewSchemaHandlerFromSchemaList(schemaElements)
res.Infos = infos
res.CreateInExMap()
return res, nil
}