jaeger之全链路追踪及span日志打印

1、安装 Jaeger UI

Jaeger 官方提供一个 all-in-one 的 docker 镜像，用于快速搭建测试环境

$ docker run -d --name jaeger \
  -e COLLECTOR_ZIPKIN_HTTP_PORT=9411 \
  -p 5775:5775/udp \
  -p 6831:6831/udp \
  -p 6832:6832/udp \
  -p 5778:5778 \
  -p 16686:16686 \
  -p 14268:14268 \
  -p 14250:14250 \
  -p 9411:9411 \
  jaegertracing/all-in-one:1.18

然后，您可以导航到http://localhost:16686以访问Jaeger UI。

2、jaeger 在项目中的使用

2.1 安装 jaeger的 Go SDK

go get -u github.com/uber/jaeger-client-go

2.2 封装 jaeger tracer 的初始化函数

// NewTracer 创建一个jaeger Tracer
func NewTracer(serviceName, addr string) (opentracing.Tracer, io.Closer, error) {
	cfg := jaegercfg.Configuration{
		ServiceName: serviceName,
		Sampler: &jaegercfg.SamplerConfig{
			Type:  jaeger.SamplerTypeConst,
			Param: 1,
		},
		Reporter: &jaegercfg.ReporterConfig{
			LogSpans:            true,
			BufferFlushInterval: 1 * time.Second,
		},
	}

	sender, err := jaeger.NewUDPTransport(addr, 0)
	if err != nil {
		return nil, nil, err
	}

	reporter := jaeger.NewRemoteReporter(sender)
	// Initialize tracer with a logger and a metrics factory
	tracer, closer, err := cfg.NewTracer(
		jaegercfg.Logger(jaeger.StdLogger),
		jaegercfg.Reporter(reporter),
	)

	return tracer, closer, err
}

2.3 在 main.go 中进行调用并设置为全局 tracer

func main() {
	tracer, io, err := utils.NewTracer(common.SRV_NAME_APIGATEWAY, common.JAEGER_ADDR)
	if err != nil {
		log.L().Bg().Fatal("unable to create tracer", zap.Error(err))
	}
	defer io.Close()
	opentracing.SetGlobalTracer(tracer)
	......
}

2.4 在API网关(gin)中使用中间件为每次请求添加SpanContexts

// TracerWrapper tracer 中间件
func TracerWrapper(c *gin.Context) {
	md := make(map[string]string)
	spanCtx, _ := opentracing.GlobalTracer().Extract(opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(c.Request.Header))
	sp := opentracing.GlobalTracer().StartSpan(c.Request.URL.Path, opentracing.ChildOf(spanCtx))
	defer sp.Finish()

	if err := opentracing.GlobalTracer().Inject(sp.Context(),
		opentracing.TextMap,
		opentracing.TextMapCarrier(md)); err != nil {
		zap.Error(err)
	}

	ctx := context.TODO()
	ctx = opentracing.ContextWithSpan(ctx, sp)
	ctx = metadata.NewContext(ctx, md)
	c.Set(contextTracerKey, ctx)

	c.Next()

	statusCode := c.Writer.Status()
	ext.HTTPStatusCode.Set(sp, uint16(statusCode))
	ext.HTTPMethod.Set(sp, c.Request.Method)
	ext.HTTPUrl.Set(sp, c.Request.URL.EscapedPath())
	if statusCode >= http.StatusInternalServerError {
		ext.Error.Set(sp, true)
	}
}

在gin路由中使用tracer中间件

func InitRouter() *gin.Engine {
	engine := gin.New()
	// 使用zap日志库
	engine.Use(ginzap.Ginzap(zap.L(), time.RFC3339, true))
	engine.Use(ginzap.RecoveryWithZap(zap.L(), true))

	g := engine.Group("/api/v1")
	g.Use(middlewares.TracerWrapper)
	{
		......
	}

详细用法请参考 jaeger-client-go相关文档及测试用例。

3、日志信息打印到span中

日志的实现参考自jaeger官方给出的样例：jaeger-examples

3.1 定义日志接口

// Logger is a simplified abstraction of the zap.Logger
type Logger interface {
	Info(msg string, fields ...zapcore.Field)
	Debug(msg string, fields ...zapcore.Field)
	Error(logToSpan bool, msg string, fields ...zapcore.Field)
	Fatal(msg string, fields ...zapcore.Field)
	With(fields ...zapcore.Field) Logger
}

3.2 定义一个普通的logger并实现3.1的接口

// logger delegates all calls to the underlying zap.Logger
type logger struct {
	logger *zap.Logger
}

// Info logs an info msg with fields
func (l logger) Info(msg string, fields ...zapcore.Field) {
	l.logger.Info(msg, fields...)
}

// Debug logs an debug msg with fields
func (l logger) Debug(msg string, fields ...zapcore.Field) {
	l.logger.Debug(msg, fields...)
}

// Error logs an error msg with fields
func (l logger) Error(logToSpan bool, msg string, fields ...zapcore.Field) {
	l.logger.Error(msg, fields...)
}

// Fatal logs a fatal error msg with fields
func (l logger) Fatal(msg string, fields ...zapcore.Field) {
	l.logger.Fatal(msg, fields...)
}

// With creates a child logger, and optionally adds some context fields to that logger.
func (l logger) With(fields ...zapcore.Field) Logger {
	return logger{logger: l.logger.With(fields...)}
}

3.3 定义打印日志到span的spanLogger，实现打印日志到span的函数，并实现3.1的接口

type spanLogger struct {
	logger *zap.Logger
	span   opentracing.Span
}

func (sl spanLogger) logToSpan(level string, msg string, fields ...zapcore.Field) {
	// TODO rather than always converting the fields, we could wrap them into a lazy logger
	fa := fieldAdapter(make([]log.Field, 0, 2+len(fields)))
	fa = append(fa, log.String("event", msg))
	fa = append(fa, log.String("level", level))
	for _, field := range fields {
		field.AddTo(&fa)
	}
	sl.span.LogFields(fa...)
}

func (sl spanLogger) Info(msg string, fields ...zapcore.Field) {
	sl.logToSpan("info", msg, fields...)
	sl.logger.Info(msg, fields...)
}

func (sl spanLogger) Debug(msg string, fields ...zapcore.Field) {
	sl.logToSpan("debug", msg, fields...)
	sl.logger.Info(msg, fields...)
}

func (sl spanLogger) Error(setErrFlag bool, msg string, fields ...zapcore.Field) {
	sl.logToSpan("error", msg, fields...)
	// 由调用者决定是否设置错误警告到span
	if setErrFlag {
		ext.Error.Set(sl.span, setErrFlag)
	}
	sl.logger.Error(msg, fields...)
}

func (sl spanLogger) Fatal(msg string, fields ...zapcore.Field) {
	sl.logToSpan("fatal", msg, fields...)
	ext.Error.Set(sl.span, true)
	sl.logger.Fatal(msg, fields...)
}

// With creates a child logger, and optionally adds some context fields to that logger.
func (sl spanLogger) With(fields ...zapcore.Field) Logger {
	return spanLogger{logger: sl.logger.With(fields...), span: sl.span}
}

3.4 实现一个工厂类(Factory)，将上述两个logger封装起来

// Factory is the default logging wrapper that can create
// logger instances either for a given Context or context-less.
type Factory struct {
	logger *zap.Logger
}

// NewFactory creates a new Factory.
func NewFactory(logger *zap.Logger) Factory {
	return Factory{logger: logger}
}

// Bg creates a context-unaware logger.
func (b Factory) Bg() Logger {
	return logger(b)
}

// For returns a context-aware Logger. If the context
// contains an OpenTracing span, all logging calls are also
// echo-ed into the span.
func (b Factory) For(ctx context.Context) Logger {
	if span := opentracing.SpanFromContext(ctx); span != nil {
		// TODO for Jaeger span extract trace/span IDs as fields
		return spanLogger{span: span, logger: b.logger}
	}
	return b.Bg()
}

// With creates a child logger, and optionally adds some context fields to that logger.
func (b Factory) With(fields ...zapcore.Field) Factory {
	return Factory{logger: b.logger.With(fields...)}
}

3.4 初始化日志组件(zap)

zap日志组件详情请参考Gin + Zap + Go Module 的结合使用。
注意，因为这里logger和spanLogger是在zap.Logger的基础上又封装了一层，所以需要配置一项：zap.AddCallerSkip(1)，否则会导致打印出来的日志行号与实际产生日志的位置不一致。

// 设置日志级别、输出格式和日志文件的路径
func SetLogs(logLevel zapcore.Level, logFormat, fileName, srvName string) {

	encoderConfig := zapcore.EncoderConfig{
		TimeKey:        TIME_KEY,
		LevelKey:       LEVLE_KEY,
		NameKey:        NAME_KEY,
		CallerKey:      CALLER_KEY,
		MessageKey:     MESSAGE_KEY,
		StacktraceKey:  STACKTRACE_KEY,
		LineEnding:     zapcore.DefaultLineEnding,
		EncodeLevel:    zapcore.CapitalLevelEncoder,    // 大写编码器
		EncodeTime:     zapcore.ISO8601TimeEncoder,     // ISO8601 UTC 时间格式
		EncodeDuration: zapcore.SecondsDurationEncoder, //
		EncodeCaller:   zapcore.ShortCallerEncoder,     // 短路径编码器(相对路径+行号)
		EncodeName:     zapcore.FullNameEncoder,
	}

	// 设置日志输出格式
	var encoder zapcore.Encoder
	switch logFormat {
	case LOGFORMAT_JSON:
		encoder = zapcore.NewJSONEncoder(encoderConfig)
	default:
		encoder = zapcore.NewConsoleEncoder(encoderConfig)
	}

	// 添加日志切割归档功能
	hook := lumberjack.Logger{
		Filename:   fileName,    // 日志文件路径
		MaxSize:    MAX_SIZE,    // 每个日志文件保存的最大尺寸 单位：M
		MaxBackups: MAX_BACKUPS, // 日志文件最多保存多少个备份
		MaxAge:     MAX_AGE,     // 文件最多保存多少天
		Compress:   true,        // 是否压缩
	}

	core := zapcore.NewCore(
		encoder, // 编码器配置
		zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stderr), zapcore.AddSync(&hook)), // 打印到控制台和文件
		zap.NewAtomicLevelAt(logLevel), // 日志级别
	)

	opts := []zap.Option{
		// 开启文件及行号
		zap.AddCaller(),
		// 开启开发模式，堆栈跟踪
		zap.Development(),
		zap.AddStacktrace(zapcore.FatalLevel),
		// 避免zap始终将包装器(wrapper)代码报告为调用方。
		zap.AddCallerSkip(1),
	}

	// 构造日志
	logger := zap.New(core, opts...)

	zap.ReplaceGlobals(logger)

	_globalL = NewFactory(zap.L().With(zap.String("service", srvName)))
}

4、日志工厂类(Factory)的应用

4.1 定义全局的Factory类型变量_globalL，并提供一个获取改全局变量的方法L()

package log

var (
	_globalL Factory
)

func L() Factory {
	return _globalL
}

4.2 日志打印到控制台或日志文件，不打印到span

... 此处省略一万字 ...

// 校验手机号码
if err := utils.ValidateMobileNumber(req.Mobile); err != nil {
	log.L().Bg().Error(false, "手机号格式错误", zap.Error(err))
	utils.AbortWithError(c, errors.ErrMobileNumberIncorrect)
	return
}

4.3 日志打印到控制台或日志文件，同时打印到span

... 此处省略一万字 ...

if !utils.VerifyPassword(req.Passwd, user.PassWord) {
	log.L().For(ctx).Error(true, "用户名或密码不正确")
	return  errors.New("用户名或密码不正确")
}

5、打印日志信息到span中，实现的效果如下

参考链接

• 官方文档
• 开源分布式跟踪系统Jaeger介绍 (六步快速上手
• 【GO-Micro】jaeger分布式链路追踪