go 问题记录（日志丢失）
问题描述：
在go程序中，通过执行一个命令启动一个子命令，并通过pipe读取子程序的标准输入和输出，通过scanner默认按行读取，此时如果子程序输出时没有携带’\n’，scanner就不会打印输出，而是会累积到缓存buf上限，最终被丢弃，直到遇到一个\n，然后输出所有的内容，默认buf缓存上限时65536，如果日志打印处还有限制，如glog就限制最大的打印字节数为4096，那么就会导致日志再次丢失。
解决方法：
不适用scanner去按行读取，直接读取管道的内容，然后设置上限，超过时或者遇到’\n’时打印
测试代码：
子程序：
#include 主程序：
package main
import (
"bufio"
"fmt"
"os/exec"
"strings"
"log"
)
func main() {
cmd := exec.Command("./test")
stdout, err := cmd.StdoutPipe()
if err != nil {
fmt.Println("Error creating StdoutPipe:", err)
return
}
cmd.Stderr = cmd.Stdout
err = cmd.Start()
if err != nil {
fmt.Println("Error starting command:", err)
return
}
scanner := bufio.NewScanner(stdout)
// scanner.Split(bufio.ScanBytes)
// buf := ""
// for scanner.Scan() {
// buf += scanner.Text()
// if strings.Contains(buf, "\n") || len(buf) >= 256 {
// log.Printf("%s", buf)
// buf = ""
// }
// }
for scanner.Scan() {
log.Printf("%s", scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Println("Error reading standard output:", err)
}
err = cmd.Wait()
if err != nil {
fmt.Println("Error waiting for command to finish:", err)
}
}
修改程序：
package main
import (
"bufio"
"fmt"
"io"
"log"
"os/exec"
)
func getReaderSize(rd io.Reader) {
b, ok := rd.(*bufio.Reader)
if ok {
log.Printf("rd size: %d", b.Size())
} else {
log.Printf("rd is not bufio.Reader")
}
}
func main() {
// Command to execute
cmd := exec.Command("./test")
// Create a pipe to capture the standard output of the command
stdout, err := cmd.StdoutPipe()
if err != nil {
fmt.Println("Error creating StdoutPipe:", err)
return
}
cmd.Stderr = cmd.Stdout
// Start the command
err = cmd.Start()
if err != nil {
fmt.Println("Error starting command:", err)
return
}
Create a scanner to read the command's standard output
//scanner := bufio.NewScanner(stdout)
//scanner.Split(bufio.ScanBytes)
//
Read and print each line from the output
//buf := make([]byte, 256)
//bufLen := 0
//for scanner.Scan() {
// buf[bufLen] = scanner.Bytes()[0]
// // buf = append(buf, scanner.Bytes()...)
// bufLen += 1
// if buf[bufLen-1] == '\n' || bufLen >= 256 {
// log.Printf("%s", string(buf[:bufLen]))
// bufLen = 0
// }
//}
//
Check for errors in scanning
//if err := scanner.Err(); err != nil {
// fmt.Println("Error reading standard output:", err)
//}
// Create a buffered reader to read from the command's stdout
reader := bufio.NewReaderSize(stdout, 256)
getReaderSize(stdout)
log.Printf("reader size: %d", reader.Size())
Buffer to store incomplete lines
//var incompleteLine []byte
//
Buffer to read chunks of bytes
//chunk := make([]byte, 256)
//
//for {
// // Read a chunk of bytes
// n, err := reader.Read(chunk)
// if err != nil {
// break // Break the loop when an error occurs (e.g., when the command finishes)
// }
//
// // Process each byte in the chunk
// for i := 0; i < n; i++ {
// b := chunk[i]
//
// // Check for newline or length exceeding 256
// if b == '\n' || len(incompleteLine) >= 256 {
// // Print the line
// log.Printf("%s", incompleteLine)
//
// // Reset the incomplete line buffer
// incompleteLine = nil
// } else {
// // Add the byte to the incomplete line buffer
// incompleteLine = append(incompleteLine, b)
// }
// }
//}
for {
s, err := reader.ReadSlice('\n')
if err != nil && err != bufio.ErrBufferFull {
if len(s) > 0 {
log.Printf("reader err but exist data, reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))
}
fmt.Println("Error reader ReadString:", err)
break // Break the loop when an error occurs (e.g., when the command finishes)
}
log.Printf("reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))
}
// Wait for the command to finish
err = cmd.Wait()
if err != nil {
fmt.Println("Error waiting for command to finish:", err)
}
}
benchmark test:
package main
import (
"strconv"
"strings"
"testing"
)
func stringTest1() string {
var buf string
for i := 0; i < 256; i++ {
buf += strconv.Itoa(i)
}
return buf
}
func stringTest2() string {
var buf strings.Builder
for i := 0; i < 256; i++ {
buf.Write([]byte(strconv.Itoa(i)))
}
return buf.String()
}
func stringTest3() string {
var buf = make([]byte, 0)
for i := 0; i < 256; i++ {
buf = append(buf, []byte(strconv.Itoa(i))...)
}
return string(buf)
}
func stringTest4() string {
var buf = make([]byte, 256)
for i := 0; i < 256; i++ {
buf[i] = '1'
}
return string(buf)
}
func BenchmarkStringTest1(b *testing.B) {
for i := 0; i < b.N; i++ {
stringTest1()
}
}
func BenchmarkStringTest2(b *testing.B) {
for i := 0; i < b.N; i++ {
stringTest2()
}
}
func BenchmarkStringTest3(b *testing.B) {
for i := 0; i < b.N; i++ {
stringTest3()
}
}
func BenchmarkStringTest4(b *testing.B) {
for i := 0; i < b.N; i++ {
stringTest4()
}
}
benchmark test
cmd:
go test -bench . -benchmem
go test -bench=<function>