日志收集系统
需求
- 实时收集,统一存储到中心系统
- 建立索引,便于搜索和查找
- 友好的web见面,通过web即可完成日志搜索
业界的elk方案
- 运维成本高,增加一个日志收集,都要改配置
- 监控确实,无法获取logstash状态
- 无法做定制开发和维护
各组件介绍
- Log Agent,日志收集客户端,用来收集服务器上的日志
- Kafka,高吞吐量的分布式队列,linkin开发,apache顶级开源项目
- ES,elasticsearch,开源的搜索引擎,提供基于http restful的web接口
- Hadoop,分布式计算框架,能够对大量数据进行分布式处理的平台
kafka应用场景
- 异步处理, 把非关键流程异步化,提高系统的响应时间和健壮性
- 流量削峰
zookeeper应用场景
log agent设计
log agent流程
kafka示例代码
package main
import (
"fmt"
"github.com/Shopify/sarama"
)
func main() {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForAll
config.Producer.Partitioner = sarama.NewRandomPartitioner
config.Producer.Return.Successes = true
msg := &sarama.ProducerMessage{}
msg.Topic = "nginx_log"
msg.Value = sarama.StringEncoder("this is a good test, my message is good")
client, err := sarama.NewSyncProducer([]string{"192.168.31.177:9092"}, config)
if err != nil {
fmt.Println("producer close, err:", err)
return
}
defer client.Close()
pid, offset, err := client.SendMessage(msg)
if err != nil {
fmt.Println("send message failed,", err)
return
}
fmt.Printf("pid:%v offset:%v\n", pid, offset)
}
tailf组件使用
package main
import (
"fmt"
"github.com/hpcloud/tail"
"time"
)
func main() {
filename := "./my.log"
tails, err := tail.TailFile(filename, tail.Config{
ReOpen: true,
Follow: true,
Location: &tail.SeekInfo{Offset: 0, Whence: 2},
MustExist: false,
Poll: true,
})
if err != nil {
fmt.Println("tail file err:", err)
return
}
var msg *tail.Line
var ok bool
for true {
msg, ok = <-tails.Lines
if !ok {
fmt.Printf("tail file close reopen, filename:%s\n", tails.Filename)
time.Sleep(100 * time.Millisecond)
continue
}
fmt.Println("msg:", msg)
}
}
配置文件库使用
iniconf, err := NewConfig("ini", "testini.conf")
if err != nil {
t.Fatal(err)
}
- 读取配置项
• String(key string) string
• Int(key string) (int, error)
• Int64(key string) (int64, error)
• Bool(key string) (bool, error)
• Float(key string) (float64, error)
package main
import (
"fmt"
"github.com/astaxie/beego/config"
)
func main() {
conf, err := config.NewConfig("ini", "./logcollect.conf")
if err != nil {
fmt.Println("new config failed, err:", err)
return
}
port, err := conf.Int("server::port")
if err != nil {
fmt.Println("read server:port failed, err:", err)
return
}
fmt.Println("Port:", port)
log_level, err := conf.Int("log::log_level")
if err != nil {
fmt.Println("read log_level failed, ", err)
return
}
fmt.Println("log_level:", log_level)
log_path := conf.String("log::log_path")
fmt.Println("log_path:", log_path)
}
日志库的使用
package main
import (
"encoding/json"
"fmt"
"github.com/astaxie/beego/logs"
)
func main() {
config := make(map[string]interface{})
config["filename"] = "./logs/logcollect.log"
config["level"] = logs.LevelDebug
configStr, err := json.Marshal(config)
if err != nil {
fmt.Println("marshal failed, err:", err)
return
}
logs.SetLogger(logs.AdapterFile, string(configStr))
logs.Debug("this is a test, my name is %s", "stu01")
logs.Trace("this is a trace, my name is %s", "stu02")
logs.Warn("this is a warn, my name is %s", "stu03")
}