Go语言实战-nginx日志处理之RabbitMQ篇
golang原生channel很好的解决多个go程之间通信和共享数据问题，但也局限于单个服务器之内，如果是多个服务器之间呢，消息队列无疑是个很好的选择。消息队列有不少，RabbitMQ、ReocketMQ、kafka等，实际使用中会根据不同的业务场景选择不同MQ。
下面简单介绍rabbitMQ的使用，还是基于之前的nginx分析处理项目，从nginx读取到的string就不存入chan中了，直接publish到RabbitMQ中，然后再起多个go程从RabbitMQ获取string进行解析处理。
// 起五个go程，消费rabbitMQ中的消息
go worker("worker1", db)
go worker("worker2", db)
go worker("worker3", db)
go worker("worker4", db)
go worker("worker5", db)
// 起一个go程，读取nginx日志，并publish到rabbitMQ
go task()task关键代码
// 声明队列
q, err := ch.QueueDeclare(
"task_queue", // name
true, // durable 是否持久化
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
// 将读取到的string，publish到rabbitMQ
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false,
amqp.Publishing{
DeliveryMode: amqp.Transient, // 指定交付模式 Persistent:轮询，Transient: 谁快谁优先
ContentType: "text/plain",
Body: []byte(body),
}
)worker关键代码
// 声明队列
q, err := ch.QueueDeclare(
"task_queue", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
// 消费消息
for d := range msgs {
parse(string(d.Body), db) //
d.Ack(false)
count++
log.Println(workerName, ": ", count)
}完整代码
package main
import (
"bufio"
"io"
"log"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/mysql"
"github.com/streadway/amqp"
)
type LogsInfo struct {
Ip string
Time int64
Method string
Path string
Protocol string
Status int
Size int
Referer string
UserAgent string
}
type LogsInfoException struct {
Ip string
Time int64
Other string
}
func main() {
db, err := gorm.Open("mysql", "root:root@/test?charset=utf8&parseTime=True&loc=Local")
defer db.Close()
if err != nil {
println("connection mysql fail")
} else {
println("connectiton mysql success")
}
// 设置全局表名禁用复数
db.SingularTable(true)
forever := make(chan bool)
// 起五个go程，消费rabbitMQ中的消息
go worker("worker1", db)
go worker("worker2", db)
go worker("worker3", db)
go worker("worker4", db)
go worker("worker5", db)
// 起一个go程，读取nginx日志，并publish到rabbitMQ
go task()
<-forever
}
// read log file and publish
func task() error {
conn, err := amqp.Dial("amqp://guest:[email protected]:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"task_queue", // name
true, // durable 是否持久化
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
// read log file
filePath := "log/access.log"
f, err := os.Open(filePath)
defer f.Close()
if err != nil {
log.Panic(err)
return err
}
buf := bufio.NewReader(f)
count := 0
for {
if count > 1000000 {
return nil
}
line, _, err := buf.ReadLine()
if err != nil {
if err == io.EOF {
return nil
}
return err
}
body := strings.TrimSpace(string(line))
// 将读取到的string入队列
err = ch.Publish(
"", // exchange
q.Name, // routing key
false, // mandatory
false,
amqp.Publishing{
DeliveryMode: amqp.Transient, // 指定交付模式 Persistent:轮询，Transient: 谁快谁优先
ContentType: "text/plain",
Body: []byte(body),
})
failOnError(err, "Failed to publish a message")
log.Println("task: ", count)
count += 1
}
}
// consumer
func worker(workerName string, db *gorm.DB) {
conn, err := amqp.Dial("amqp://guest:[email protected]:5672/")
failOnError(err, "Failed to connect to RabbitMQ")
defer conn.Close()
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
q, err := ch.QueueDeclare(
"task_queue", // name
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
failOnError(err, "Failed to declare a queue")
err = ch.Qos(
1, // prefetch count
0, // prefetch size
false, // global
)
failOnError(err, "Failed to set QoS")
// consumer
msgs, err := ch.Consume(
q.Name, // queue
"", // consumer
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
failOnError(err, "Failed to register a consumer")
forever := make(chan bool)
count := 0
go func() {
for d := range msgs {
parse(string(d.Body), db) //
d.Ack(false)
count++
log.Println(workerName, ": ", count)
}
}()
log.Printf(" [*] Waiting for messages. To exit press CTRL+C")
<-forever
}
// parse and write
func parse(str string, db *gorm.DB) {
re := `^([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}) - - \[(.*)\] "([^\s]+) ([^\s]+) ([^\s]+?)" ([\d]{3}) ([\d]{1,9}) "([^"]*?)" "([^"]*?)"`
reg := regexp.MustCompile(re)
parseInfo := reg.FindStringSubmatch(str)
if len(parseInfo) == 0 {
re1 := `^([\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}) - - \[(.*)\] (.*)`
reg1 := regexp.MustCompile(re1)
parseInfo1 := reg1.FindStringSubmatch(str)
if len(parseInfo1) == 0 {
return
}
t1, _ := time.Parse("02/Jan/2006:15:04:05 -0700", parseInfo1[2])
infoException := LogsInfoException{
Ip: parseInfo1[1],
Time: t1.Unix(),
Other: parseInfo1[3],
}
db.Create(&infoException)
return
}
t, _ := time.Parse("02/Jan/2006:15:04:05 -0700", parseInfo[2])
status, _ := strconv.Atoi(parseInfo[6])
size, _ := strconv.Atoi(parseInfo[7])
info := LogsInfo{
Ip: parseInfo[1],
Time: t.Unix(),
Method: parseInfo[3],
Path: parseInfo[4],
Protocol: parseInfo[5],
Status: status,
Size: size,
Referer: parseInfo[8],
UserAgent: parseInfo[9],
}
db.Create(&info)
}
func failOnError(err error, msg string) {
if err != nil {
log.Fatalf("%s: %s", msg, err)
}
}
demo已放码云，有兴趣的可以clone到本地跑