Go自研微服务框架-参数处理
参数处理
1. 频繁创建context的优化
sync.Pool用于存储那些被分配了但是没有被使用，但是未来可能被使用的值，这样可以不用再次分配内存，提高效率。
sync.Pool大小是可伸缩的，高负载是会动态扩容，存放在池中不活跃的对象会被自动清理。
type Engine struct {
*router
funcMap template.FuncMap
HTMLRender render.HTMLRender
pool sync.Pool
}
func New() *Engine {
engine := &Engine{
router: &router{},
funcMap: nil,
HTMLRender: render.HTMLRender{},
}
engine.pool.New = func() any {
return engine.allocateContext()
}
return engine
}
func (e *Engine) allocateContext() any {
return &Context{engin: e}
}
func (e *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx := e.pool.Get().(*Context)
ctx.W = w
ctx.R = r
e.handleHttpRequest(ctx)
e.pool.Put(ctx)
}
func (e *Engine) handleHttpRequest(ctx *Context) {
groups := e.router.groups
for _, g := range groups {
uri := ctx.R.RequestURI
routerName := SubStringLast(uri, "/"+g.groupName)
node := g.treeNode.Get(routerName)
if node == nil || !node.isEnd {
ctx.W.WriteHeader(http.StatusNotFound)
fmt.Fprintln(ctx.W, ctx.R.RequestURI+" not found")
return
}
methodHandle, ok := g.handlerMap[routerName]
if ok {
_, ok = methodHandle[ANY]
if ok {
g.methodHandle(ANY, methodHandle, ctx)
return
}
_, ok = methodHandle[ctx.R.Method]
if ok {
g.methodHandle(ctx.R.Method, methodHandle, ctx)
return
}
ctx.W.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintln(ctx.W, ctx.R.Method+" not allowed")
}
}
}
2. query参数
首先我们来处理query参数，比如：
http://xxx.com/user/add?id=1&age=20&username=张三记得将路由的URL匹配改为：
uri := ctx.R.URL.Path
type Context struct {
W http.ResponseWriter
R *http.Request
engin *Engine
queryCache url.Values
}
func (c *Context) GetDefaultQuery(key string, defaultValue string) string {
values, ok := c.GetQueryArray(key)
if !ok {
return defaultValue
} else {
return values[0]
}
}
func (c *Context) GetQuery(key string) string {
c.initQueryCache()
return c.queryCache.Get(key)
}
func (c *Context) QueryArray(key string) (values []string) {
c.initQueryCache()
values, _ = c.queryCache[key]
return
}
func (c *Context) GetQueryArray(key string) (values []string, ok bool) {
c.initQueryCache()
values, ok = c.queryCache[key]
return
}
func (c *Context) initQueryCache() {
if c.queryCache == nil {
if c.R != nil {
c.queryCache = c.R.URL.Query()
} else {
c.queryCache = url.Values{}
}
}
}
2.1 map类型参数
类似于http://localhost:8080/queryMap?user[id]=1&user[name]=张三
func (c *Context) QueryMap(key string) (dicts map[string]string) {
dicts, _ = c.GetQueryMap(key)
return
}
func (c *Context) GetQueryMap(key string) (map[string]string, bool) {
c.initQueryCache()
return c.get(c.queryCache, key)
}
func (c *Context) get(m map[string][]string, key string) (map[string]string, bool) {
//user[id]=1&user[name]=张三
dicts := make(map[string]string)
exist := false
for k, value := range m {
if i := strings.IndexByte(k, '['); i >= 1 && k[0:i] == key {
if j := strings.IndexByte(k[i+1:], ']'); j >= 1 {
exist = true
dicts[k[i+1:][:j]] = value[0]
}
}
}
return dicts, exist
}
3. Post表单参数
获取表单参数我们借助 http.Request.PostForm
Form属性包含了post表单和url后面跟的get参数。
PostForm属性只包含了post表单参数。
func (c *Context) initFormCache() {
if c.formCache == nil {
c.formCache = make(url.Values)
req := c.R
if err := req.ParseMultipartForm(defaultMultipartMemory); err != nil {
if !errors.Is(err, http.ErrNotMultipart) {
log.Println(err)
}
}
c.formCache = c.R.PostForm
}
}
func (c *Context) GetPostForm(key string) (string, bool) {
if values, ok := c.GetPostFormArray(key); ok {
return values[0], ok
}
return "", false
}
func (c *Context) PostFormArray(key string) (values []string) {
values, _ = c.GetPostFormArray(key)
return
}
func (c *Context) GetPostFormArray(key string) (values []string, ok bool) {
c.initFormCache()
values, ok = c.formCache[key]
return
}
func (c *Context) GetPostFormMap(key string) (map[string]string, bool) {
c.initFormCache()
return c.get(c.formCache, key)
}
func (c *Context) PostFormMap(key string) (dicts map[string]string) {
dicts, _ = c.GetPostFormMap(key)
return
}
4. 文件参数
借助http.Request.FormFile
func (c *Context) FormFile(name string) (*multipart.FileHeader, error) {
req := c.R
if err := req.ParseMultipartForm(defaultMultipartMemory); err != nil {
return nil, err
}
file, header, err := req.FormFile(name)
if err != nil {
return nil, err
}
err = file.Close()
if err != nil {
return nil, err
}
return header, nil
}
g.Post("/file", func(ctx *msgo.Context) {
file, err := ctx.FormFile("file")
if err != nil {
log.Println(err)
}
src, err := file.Open()
defer src.Close()
if err != nil {
log.Println(err)
} else {
out, err := os.Create("./upload/test.png")
defer out.Close()
if err != nil {
log.Println(err)
} else {
io.Copy(out, src)
}
}
})
一般会有将文件存储的需求，可以将上述的代码也提取为一个方法
func (c *Context) SaveUploadedFile(file *multipart.FileHeader, dst string) error {
src, err := file.Open()
if err != nil {
return err
}
defer src.Close()
out, err := os.Create(dst)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, src)
return err
}
func (c *Context) MultipartForm() (*multipart.Form, error) {
err := c.R.ParseMultipartForm(defaultMultipartMemory)
return c.R.MultipartForm, err
}
5. json参数
现在流行前后端分离开发，前后端交互以json的形式来通信
json参数：
- content-type: application/json
- post传参
一般在写代码时，我们期望这样的处理方式：
g.Post("/jsonParam", func(ctx *msgo.Context) {
user := &User{}
err := ctx.DealJson(user)
if err == nil {
ctx.JSON(http.StatusOK, user)
} else {
log.Println(err)
}
})
func (c *Context) DealJson(data any) error {
body := c.R.Body
if c.R == nil || body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
return decoder.Decode(data)
}
测试：
{
"name":"张三",
"age":10,
"addresses":[
"北京",
"杭州"
]
}
[
{
"name":"11",
"age":10,
"addresses":[
"北京"
]
}
]
如果想要实现参数中有的属性，但是对应的结构体没有，报错，也就是检查结构体是否有效
decoder.DisallowUnknownFields()
如果结构体有的属性，但是参数中没有，想要校验这种错误，又该如何做呢？
5.1 结构体校验
如果想要达到上述的效果，我们可以写一个校验器，专门来处理这样的校验。
为了实现这种效果，我们可以改变一下思路，先将所有的参数解析为map，然后和对应的结构体进行比对
func (c *Context) DealJson(data any) error {
body := c.R.Body
if c.R == nil || body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
if c.DisallowUnknownFields {
decoder.DisallowUnknownFields()
}
if c.IsValidate {
err := validateRequireParam(data, decoder)
if err != nil {
return err
}
} else {
err := decoder.Decode(data)
if err != nil {
return err
}
}
return nil
}
func validateRequireParam(data any, decoder *json.Decoder) error {
if data == nil {
return nil
}
valueOf := reflect.ValueOf(data)
if valueOf.Kind() != reflect.Pointer {
return errors.New("no ptr type")
}
t := valueOf.Elem().Interface()
of := reflect.ValueOf(t)
switch of.Kind() {
case reflect.Struct:
mapData := make(map[string]interface{})
_ = decoder.Decode(&mapData)
for i := 0; i < of.NumField(); i++ {
field := of.Type().Field(i)
tag := field.Tag.Get("json")
value := mapData[tag]
if value == nil {
return errors.New(fmt.Sprintf("filed [%s] is not exist", tag))
}
}
marshal, _ := json.Marshal(mapData)
_ = json.Unmarshal(marshal, data)
}
return nil
}
如果并不是所有字段都是必须的呢?
type User struct {
Name string `xml:"name" json:"name" msgo:"required"`
Age int `xml:"name" json:"age"`
Addresses []string `json:"addresses"`
}
for i := 0; i < of.NumField(); i++ {
field := of.Type().Field(i)
required := field.Tag.Get("msgo")
tag := field.Tag.Get("json")
value := mapData[tag]
if value == nil && required == "required" {
return errors.New(fmt.Sprintf("filed [%s] is required", tag))
}
}
5.2 切片数组校验
func (c *Context) DealJson(data any) error {
body := c.R.Body
if c.R == nil || body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
if c.DisallowUnknownFields {
decoder.DisallowUnknownFields()
}
if c.IsValidate {
err := validateRequireParam(data, decoder)
if err != nil {
return err
}
} else {
err := decoder.Decode(data)
if err != nil {
return err
}
}
return nil
}
func validateRequireParam(data any, decoder *json.Decoder) error {
if data == nil {
return nil
}
valueOf := reflect.ValueOf(data)
if valueOf.Kind() != reflect.Pointer {
return errors.New("no ptr type")
}
t := valueOf.Elem().Interface()
of := reflect.ValueOf(t)
switch of.Kind() {
case reflect.Struct:
return checkParam(of, data, decoder)
case reflect.Slice, reflect.Array:
elem := of.Type().Elem()
elemType := elem.Kind()
if elemType == reflect.Struct {
return checkParamSlice(elem, data, decoder)
}
default:
err := decoder.Decode(data)
if err != nil {
return err
}
}
return nil
}
func checkParamSlice(elem reflect.Type, data any, decoder *json.Decoder) error {
mapData := make([]map[string]interface{}, 0)
_ = decoder.Decode(&mapData)
if len(mapData) <= 0 {
return nil
}
for i := 0; i < elem.NumField(); i++ {
field := elem.Field(i)
required := field.Tag.Get("msgo")
tag := field.Tag.Get("json")
value := mapData[0][tag]
if value == nil && required == "required" {
return errors.New(fmt.Sprintf("filed [%s] is required", tag))
}
}
if data != nil {
marshal, _ := json.Marshal(mapData)
_ = json.Unmarshal(marshal, data)
}
return nil
}
5.3 引入第三方校验
gin等框架在做校验时，是使用了
https://github.com/go-playground/validator组件，我们也将其集成进来
type User struct {
Name string `xml:"name" json:"name" msgo:"required"`
Age int `xml:"name" json:"age" validate:"required,max=50,min=18"`
Addresses []string `json:"addresses"`
}
func (c *Context) DealJson(data any) error {
body := c.R.Body
if c.R == nil || body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
if c.DisallowUnknownFields {
decoder.DisallowUnknownFields()
}
if c.IsValidate {
err := validateRequireParam(data, decoder)
if err != nil {
return err
}
} else {
err := decoder.Decode(data)
if err != nil {
return err
}
}
return validate(data)
}
type SliceValidationError []error
func (err SliceValidationError) Error() string {
n := len(err)
switch n {
case 0:
return ""
default:
var b strings.Builder
if err[0] != nil {
fmt.Fprintf(&b, "[%d]: %s", 0, err[0].Error())
}
if n > 1 {
for i := 1; i < n; i++ {
if err[i] != nil {
b.WriteString("\n")
fmt.Fprintf(&b, "[%d]: %s", i, err[i].Error())
}
}
}
return b.String()
}
}
func validate(obj any) error {
if obj == nil {
return nil
}
value := reflect.ValueOf(obj)
switch value.Kind() {
case reflect.Ptr:
return validate(value.Elem().Interface())
case reflect.Struct:
return validateStruct(obj)
case reflect.Slice, reflect.Array:
count := value.Len()
validateRet := make(SliceValidationError, 0)
for i := 0; i < count; i++ {
if err := validateStruct(value.Index(i).Interface()); err != nil {
validateRet = append(validateRet, err)
}
}
if len(validateRet) == 0 {
return nil
}
return validateRet
default:
return nil
}
}
func validateStruct(obj any) error {
return validator.New().Struct(obj)
}
6. 优化验证器-接口+单例
在上面 验证的时候，每次都需要使用
validator.New()，会极大的浪费性能，可以使用单例来做优化，同时验证器的实现可能有多种，提供接口，便于扩展
type StructValidator interface {
//结构体验证，如果错误返回对应的错误信息
ValidateStruct(any) error
//返回对应使用的验证器
Engine() any
}
package msgo
import (
"github.com/go-playground/validator/v10"
"reflect"
"sync"
)
type StructValidator interface {
// ValidateStruct 结构体验证，如果错误返回对应的错误信息
ValidateStruct(any) error
// Engine 返回对应使用的验证器
Engine() any
}
type defaultValidator struct {
one sync.Once
validate *validator.Validate
}
func (d *defaultValidator) ValidateStruct(obj any) error {
if obj == nil {
return nil
}
value := reflect.ValueOf(obj)
switch value.Kind() {
case reflect.Ptr:
return d.ValidateStruct(value.Elem().Interface())
case reflect.Struct:
return d.validateStruct(obj)
case reflect.Slice, reflect.Array:
count := value.Len()
validateRet := make(SliceValidationError, 0)
for i := 0; i < count; i++ {
if err := d.validateStruct(value.Index(i).Interface()); err != nil {
validateRet = append(validateRet, err)
}
}
if len(validateRet) == 0 {
return nil
}
return validateRet
default:
return nil
}
}
func (d *defaultValidator) validateStruct(obj any) error {
d.lazyInit()
return d.validate.Struct(obj)
}
func (d *defaultValidator) lazyInit() {
d.one.Do(func() {
d.validate = validator.New()
})
}
func (c *Context) DealJson(data any) error {
body := c.R.Body
if c.R == nil || body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
if c.DisallowUnknownFields {
decoder.DisallowUnknownFields()
}
if c.IsValidate {
err := validateRequireParam(data, decoder)
if err != nil {
return err
}
} else {
err := decoder.Decode(data)
if err != nil {
return err
}
}
return validate(data)
}
func validate(obj any) error {
return Validator.ValidateStruct(obj)
}
7. 多种类型参数接收-绑定器实现
在实际中，我们需要处理json参数，xml参数或者其他参数，同样，我们将其行为抽象为接口，赋予其不同实现，这样代码更加优雅，扩展维护更加方便
import "net/http"
type Binding interface {
Name() string
Bind(*http.Request, any) error
}
7.1 JSON绑定器
package binding
import "net/http"
type Binding interface {
Name() string
Bind(*http.Request, any) error
}
var JSON = jsonBinding{}
package bind
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
)
type jsonBinding struct {
DisallowUnknownFields bool
IsValidate bool
}
func (b jsonBinding) Name() string {
return "json"
}
func (b jsonBinding) Bind(r *http.Request, obj any) error {
body := r.Body
//post 传参的内容 是放在body中的
if body == nil {
return errors.New("invalid request")
}
decoder := json.NewDecoder(body)
if b.DisallowUnknownFields {
decoder.DisallowUnknownFields()
}
if b.IsValidate {
err := validateParam(obj, decoder)
if err != nil {
return err
}
} else {
err := decoder.Decode(obj)
if err != nil {
return err
}
}
return validate(obj)
}
func validateParam(obj any, decoder *json.Decoder) error {
//反射
valueOf := reflect.ValueOf(obj)
//判断其是否为指针类型
if valueOf.Kind() != reflect.Pointer {
return errors.New("This argument must have a pointer type")
}
elem := valueOf.Elem().Interface()
of := reflect.ValueOf(elem)
switch of.Kind() {
case reflect.Struct:
return checkParam(of, obj, decoder)
case reflect.Slice, reflect.Array:
elem := of.Type().Elem()
if elem.Kind() == reflect.Struct {
return checkParamSlice(elem, obj, decoder)
}
default:
_ = decoder.Decode(obj)
}
return nil
}
func checkParamSlice(of reflect.Type, obj any, decoder *json.Decoder) error {
mapValue := make([]map[string]interface{}, 0)
_ = decoder.Decode(&mapValue)
for i := 0; i < of.NumField(); i++ {
field := of.Field(i)
name := field.Name
jsonName := field.Tag.Get("json")
if jsonName != "" {
name = jsonName
}
required := field.Tag.Get("msgo")
for _, v := range mapValue {
value := v[name]
if value == nil && required == "required" {
return errors.New(fmt.Sprintf("filed [%s] is not exist,because [%s] is required", jsonName, jsonName))
}
}
}
b, _ := json.Marshal(mapValue)
_ = json.Unmarshal(b, obj)
return nil
}
func checkParam(of reflect.Value, obj any, decoder *json.Decoder) error {
//解析为map，然后根据map中的key 进行比对
//判断类型 结构体 才能解析为map
mapValue := make(map[string]interface{})
_ = decoder.Decode(&mapValue)
for i := 0; i < of.NumField(); i++ {
field := of.Type().Field(i)
name := field.Name
jsonName := field.Tag.Get("json")
if jsonName != "" {
name = jsonName
}
required := field.Tag.Get("msgo")
value := mapValue[name]
if value == nil && required == "required" {
return errors.New(fmt.Sprintf("filed [%s] is not exist,because [%s] is required", jsonName, jsonName))
}
}
b, _ := json.Marshal(mapValue)
_ = json.Unmarshal(b, obj)
return nil
}
func (c *Context) BindJson(obj any) error {
jsonBinding := binding.JSON
jsonBinding.DisallowUnknownFields = c.DisallowUnknownFields
jsonBinding.IsValidate = c.IsValidate
return c.MustBindWith(obj, jsonBinding)
}
func (c *Context) MustBindWith(obj any, bind bind.Binding) error {
if err := c.ShouldBind(obj, bind); err != nil {
c.W.WriteHeader(http.StatusBadRequest)
return err
}
return nil
}
func (c *Context) ShouldBind(obj any, bind bind.Binding) error {
return bind.Bind(c.R, obj)
}
7.2 XML绑定器
func (c *Context) BindXML(obj any) error {
return c.MustBindWith(obj, binding.XML)
}
package bind
import (
"encoding/xml"
"net/http"
)
type xmlBinding struct {
}
func (b xmlBinding) Name() string {
return "mxl"
}
func (b xmlBinding) Bind(r *http.Request, obj any) error {
if r.Body == nil {
return nil
}
decoder := xml.NewDecoder(r.Body)
if err := decoder.Decode(obj); err != nil {
return err
}
return validate(obj)
}
type User struct {
Name string `xml:"name" json:"name" msgo:"required"`
Age int `xml:"age" json:"age" validate:"required,max=50,min=18"`
Addresses []string `json:"addresses"`
}
<User>
<name>张三name>
<age>20age>
User>
g.Post("/xmlParam", func(ctx *msgo.Context) {
user := &User{}
//user := User{}
err := ctx.BindXML(user)
if err == nil {
ctx.JSON(http.StatusOK, user)
} else {
log.Println(err)
}
})