区块链项目：golang内存泄漏，goroutine泄漏排查
一、pprof的使用
##pprof的启动及操作
import _ "net/http/pprof"
go func() {
http.ListenAndServe("0.0.0.0:11181", nil)
}()
然后通过网页打开即可：http://192.168.9.78:11181/debug/pprof/
pprof的一些操作：
go tool pprof -inuse_space http://192.168.9.78:11181/debug/pprof/heap
##pprof信息的解读：
###/debug/pprof/goroutine:
goroutine profile: total 63
4 @ 0x4382fa 0x433417 0x432a59 0x539888 0x5398f4 0x53b047 0x54ee20 0x643e08 0x644374 0x64807d 0x57dcf2 0x47aba9 0x47ad18 0xafc78b 0xafd0e4 0x119c18f 0x118eb7d 0x465691
# 0x432a58 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x539887 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x5398f3 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x53b046 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x54ee1f net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x643e07 crypto/tls.(*block).readFromUntil+0x97 /usr/local/go/src/crypto/tls/conn.go:488
# 0x644373 crypto/tls.(*Conn).readRecord+0xc3 /usr/local/go/src/crypto/tls/conn.go:590
# 0x64807c crypto/tls.(*Conn).Read+0x11c /usr/local/go/src/crypto/tls/conn.go:1134
# 0x57dcf1 bufio.(*Reader).Read+0x311 /usr/local/go/src/bufio/bufio.go:213
# 0x47aba8 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x47ad17 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0xafc78a Masami/backEnd/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:237
# 0xafd0e3 Masami/backEnd/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:492
# 0x119c18e Masami/backEnd/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http_util.go:608
# 0x118eb7c Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xcc /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1115
4 @ 0x4382fa 0x447704 0x44636c 0x118fa7a 0x465691
# 0x118fa79 Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).controller+0x659 /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1188
####解读：
协程总数：
下面信息列出了每个协程运行的信息
每个协程的总数量@协程运行的位置
###/debug/pprof/heap:
heap profile: 20: 1547872 [155297: 2755898608] @ heap/1048576
1: 1384448 [2: 2768896] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
# 0x775a9e runtime/pprof.writeHeap+0x8e /usr/local/go/src/runtime/pprof/pprof.go:489
# 0x773b44 runtime/pprof.(*Profile).WriteTo+0x3b4 /usr/local/go/src/runtime/pprof/pprof.go:302
# 0xa10570 net/http/pprof.handler.ServeHTTP+0x1d0 /usr/local/go/src/net/http/pprof/pprof.go:209
# 0xa10882 net/http/pprof.Index+0x1e2 /usr/local/go/src/net/http/pprof/pprof.go:221
# 0x6e8b03 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
# 0x6e9f3f net/http.(*ServeMux).ServeHTTP+0x12f /usr/local/go/src/net/http/server.go:2238
# 0x6eb301 net/http.serverHandler.ServeHTTP+0x91 /usr/local/go/src/net/http/server.go:2568
# 0x6e7671 net/http.(*conn).serve+0x611 /usr/local/go/src/net/http/server.go:1825
1: 155648 [1: 155648] @ 0x609d3f 0x47957e 0x60a3bb 0x608925 0x61ab8e 0x619ada 0x933fe8 0x934590 0x952e9e 0x950540 0x958d10 0x167b977 0x167b10b 0x17b61bc 0x1792cb5 0x17d8994 0x465691
# 0x609d3e crypto/elliptic.initTable+0x3e /usr/local/go/src/crypto/elliptic/p256_amd64.go:382
# 0x47957d sync.(*Once).Do+0xbd /usr/local/go/src/sync/once.go:44
# 0x60a3ba crypto/elliptic.(*p256Point).p256BaseMult+0x4a /usr/local/go/src/crypto/elliptic/p256_amd64.go:426
# 0x608924 crypto/elliptic.p256Curve.ScalarBaseMult+0xc4 /usr/local/go/src/crypto/elliptic/p256_amd64.go:240
# 0x61ab8d crypto/x509.parseECPrivateKey+0x41d /usr/local/go/src/crypto/x509/sec1.go:102
# 0x619ad9 crypto/x509.ParsePKCS8PrivateKey+0x369 /usr/local/go/src/crypto/x509/pkcs8.go:45
# 0x933fe7 Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils.DERToPrivateKey+0x77 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils/keys.go:192
# 0x93458f Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils.PEMtoPrivateKey+0x38f /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/utils/keys.go:237
# 0x952e9d Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*fileBasedKeyStore).loadPrivateKey+0x3ad /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/fileks.go:338
# 0x95053f Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*fileBasedKeyStore).GetKey+0x4bf /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/fileks.go:134
# 0x958d0f Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw.(*impl).GetKey+0x6f /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/bccsp/sw/impl.go:255
# 0x167b976 Masami/backEnd/vendor/ILIOS/core/fabricsdk.getDefaultImplPreEnrolledUser+0x7d6 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/fabricsdk/sdkutil.go:107
# 0x167b10a Masami/backEnd/vendor/ILIOS/core/fabricsdk.GetAdmin+0x24a /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/fabricsdk/sdkutil.go:49
# 0x17b61bb Masami/backEnd/vendor/ILIOS/core/blockchain.setupClient+0x16b /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainSdkConfig.go:391
# 0x1792cb4 Masami/backEnd/vendor/ILIOS/core/blockchain.setChannel+0x54 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1212
# 0x17d8993 Masami/backEnd/vendor/ILIOS/core/blockchain.(*BlockchainChannel).QueryBlock.func1+0xb3 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1231
.....
# runtime.MemStats
# Alloc = 16831912
# TotalAlloc = 93465363144
# Sys = 111851768
# Lookups = 436597
# Mallocs = 2411037775
# Frees = 2410767219
# HeapAlloc = 16831912
# HeapSys = 30670848
# HeapIdle = 5324800
# HeapInuse = 25346048
# HeapReleased = 0
# HeapObjects = 270556
# Stack = 1179648 / 1179648
# MSpan = 415720 / 425984
# MCache = 4800 / 16384
# BuckHashSys = 2750142
# GCSys = 75923456
# OtherSys = 885306
# NextGC = 22628176
# PauseNs = [251625 102144 1114973 82811 962412 55545 70193 71273 83591 2671351 80293 61109 95033 78908 131701 59840 84397 857675 129996 1180145 60259 104967 54462 92301 63168 87525 3775060 67801 98686 75025 52108 4226429 69942 83824 101308 100086 222931 85903 141632 108922 49959 77062 41757 111464 91770 653235 83308 55181 87047 86229 78250 66959 128160 69158 83498 81554 91212 104690 81473 98758 66789 170286 103240 66697 85881 55563 119912 157109 115442 82078 98494 130530 113911 123343 45100 144668 3457395 111751 76875 100159 87038 87886 90464 118198 3193192 105136 112051 73821 61658 102659 79153 62140 100095 75005 71233 8313802 238423 70598 57186 801672 84199 69673 1850699 352710 372426 80204 91827 88774 105581 168779 117494 3621083 45458 96822 80238 83728 87826 107449 68569 109294 3241219 71098 3377295 76956 3705284 187921 3296415 3589375 128336 125871 126073 118386 84267 150214 68185 78886 180731 65156 102083 57375 83865 84446 1795706 1987823 81895 452550 75960 84030 159927 88024 77141 64582 154597 106656 91543 3260111 101412 63701 102325 71723 104087 67210 119190 74439 73354 126235 63346 101999 78577 76937 100137 60490 57165 149083 87388 248331 115907 82016 101468 97311 89321 114089 242282 103262 180517 88439 94698 5247411 248777 79423 105661 111279 91862 202097 103120 64497 62549 101927 77345 107139 61002 3071591 68166 101282 1082715 73638 893508 116980 121683 149156 65585 78968 522928 51675 4243053 128256 130818 4473130 78633 163665 230193 123923 93318 96058 63285 91195 2865382 3166808 69171 88237 106198 58686 1892996 92479 74520 95618 887626 426498 87927 118181 108641 116303 126501 78906 100257 144567 3517513 101324 106846 123704 82077 2957290 104086 89691 108962 223224]
# NumGC = 13036
# DebugGC = false
####解读：
heap profile: 20(正在使用的对象数量): 1547872（正在使用的内存大小：bytes） [155297(分配的对象数量): 2755898608（分配的内存大小）] @ heap/1048576
下面信息列出了每个对象的具体使用情况
1: 1384448 [2: 2768896]（与上面的含义一样，分别是：正在使用的对象数量， 正在使用的内存大小；分配的对象数量，分配的内存大小） @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691（栈指针地址）
第二部分意思：是runtime.memstats的统计信息
二、内存泄漏，goroutine泄漏排查：
#背景：
系统在做稳定性测试时发现内存持续很快的增长，初步判断有泄漏风险
#排查步骤：
##一、打开pprof监测程序
###堆情况：
heap profile: 66: 2958232 [282858: 9664223888] @ heap/1048576
1: 1368064 [1: 1368064] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
# 0x775a9e runtime/pprof.writeHeap+0x8e /usr/local/go/src/runtime/pprof/pprof.go:489
# 0x773b44 runtime/pprof.(*Profile).WriteTo+0x3b4 /usr/local/go/src/runtime/pprof/pprof.go:302
# 0xa10570 net/http/pprof.handler.ServeHTTP+0x1d0 /usr/local/go/src/net/http/pprof/pprof.go:209
# 0xa10882 net/http/pprof.Index+0x1e2 /usr/local/go/src/net/http/pprof/pprof.go:221
# 0x6e8b03 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
# 0x6e9f3f net/http.(*ServeMux).ServeHTTP+0x12f /usr/local/go/src/net/http/server.go:2238
# 0x6eb301 net/http.serverHandler.ServeHTTP+0x91 /usr/local/go/src/net/http/server.go:2568
# 0x6e7671 net/http.(*conn).serve+0x611 /usr/local/go/src/net/http/server.go:1825
1: 1359872 [2: 2719744] @ 0x775a9f 0x773b45 0xa10571 0xa10883 0x6e8b04 0x6e9f40 0x6eb302 0x6e7672 0x465691
# 0x775a9e runtime/pprof.writeHeap+0x8e /usr/local/go/src/runtime/pprof/pprof.go:489
# 0x773b44 runtime/pprof.(*Profile).WriteTo+0x3b4 /usr/local/go/src/runtime/pprof/pprof.go:302
# 0xa10570 net/http/pprof.handler.ServeHTTP+0x1d0 /usr/local/go/src/net/http/pprof/pprof.go:209
# 0xa10882 net/http/pprof.Index+0x1e2 /usr/local/go/src/net/http/pprof/pprof.go:221
# 0x6e8b03 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
# 0x6e9f3f net/http.(*ServeMux).ServeHTTP+0x12f /usr/local/go/src/net/http/server.go:2238
# 0x6eb301 net/http.serverHandler.ServeHTTP+0x91 /usr/local/go/src/net/http/server.go:2568
# 0x6e7671 net/http.(*conn).serve+0x611 /usr/local/go/src/net/http/server.go:1825
1: 90112 [1: 90112] @ 0x413ca6 0x7db566 0x7f0ced 0x7fc74e 0x84dcf3 0x8518d8 0x924a90 0x163ef7b 0x17e0068 0x17e32f4 0x191b259 0x437e6a 0x465691
# 0x7db565 html.init+0x75 /usr/local/go/src/html/entity.go:16
# 0x7f0cec html/template.init+0x7c /usr/local/go/src/html/template/url.go:106
# 0x7fc74d Masami/backEnd/vendor/github.com/astaxie/beego/context.init+0xcd /opt/gopath/src/Masami/backEnd/vendor/github.com/astaxie/beego/context/output.go:349
# 0x84dcf2 Masami/backEnd/vendor/github.com/astaxie/beego.init+0xc2 /opt/gopath/src/Masami/backEnd/vendor/github.com/astaxie/beego/tree.go:581
# 0x8518d7 Masami/backEnd/vendor/ILIOS/common/packager.init+0x77 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/common/packager/packager.go:38
# 0x924a8f Masami/backEnd/vendor/ILIOS/common.init+0x4f /opt/gopath/src/Masami/backEnd/vendor/ILIOS/common/utils.go:82
# 0x163ef7a Masami/backEnd/vendor/ILIOS/core.init+0x4a /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/zkInstance.go:265
# 0x17e0067 Masami/backEnd/vendor/ILIOS/core/blockchain.init+0x47 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/util.go:366
# 0x17e32f3 Masami/backEnd/vendor/ILIOS/ilioslaunch.init+0x43 /opt/gopath/src/Masami/backEnd/vendor/ILIOS/ilioslaunch/launch.go:14
# 0x191b258 main.init+0x48 /opt/gopath/src/Masami/backEnd/main.go:47
# 0x437e69 runtime.main+0x1c9 /usr/local/go/src/runtime/proc.go:173
22: 70400 [65: 208000] @ 0x110b612 0x110eff7 0x110c6ff 0x110ff0c 0x110c6ff 0x110c50e 0x110c190 0x11cb597 0x110cf0d 0x110c50e 0x11af4a9 0x11c34f7 0x11b21bf 0x11bcce2 0x11cc442 0x1665d0c 0x465691
# 0x110b611 Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes+0x171 /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:298
# 0x110eff6 Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte+0x36 /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:802
# 0x110c6fe Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
# 0x110ff0b Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_struct_message+0x13b /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:920
# 0x110c6fe Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
# 0x110c50d Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
# 0x110c18f Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeMessage+0xbf /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:421
# 0x11cb596 Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer._Event_OneofUnmarshaler+0x216 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:432
# 0x110cf0c Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x9cc /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:508
# 0x110c50d Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
# 0x11af4a8 Masami/backEnd/vendor/google.golang.org/grpc.protoCodec.Unmarshal+0x118 /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/codec.go:85
# 0x11c34f6 Masami/backEnd/vendor/google.golang.org/grpc.(*protoCodec).Unmarshal+0x76 :35
# 0x11b21be Masami/backEnd/vendor/google.golang.org/grpc.recv+0x16e /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/rpc_util.go:382
# 0x11bcce1 Masami/backEnd/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1 /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/stream.go:398
# 0x11cc441 Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer.(*eventsChatClient).Recv+0x61 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:559
# 0x1665d0b Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/consumer.(*eventsClient).processEvents+0xab /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/consumer/consumer.go:236
19: 19456 [62: 63488] @ 0x110b612 0x110eff7 0x110c6ff 0x110ff0c 0x110c6ff 0x110c50e 0x110c190 0x11cb597 0x110cf0d 0x110c50e 0x11af4a9 0x11c34f7 0x11b21bf 0x11bcce2 0x11cc442 0x1665d0c 0x465691
# 0x110b611 Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes+0x171 /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:298
# 0x110eff6 Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte+0x36 /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:802
# 0x110c6fe Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
# 0x110ff0b Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_struct_message+0x13b /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:920
# 0x110c6fe Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x1be /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:537
# 0x110c50d Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
# 0x110c18f Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeMessage+0xbf /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:421
# 0x11cb596 Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer._Event_OneofUnmarshaler+0x216 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:432
# 0x110cf0c Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).unmarshalType+0x9cc /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:508
# 0x110c50d Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).Unmarshal+0x1cd /opt/gopath/src/Masami/backEnd/vendor/github.com/golang/protobuf/proto/decode.go:452
# 0x11af4a8 Masami/backEnd/vendor/google.golang.org/grpc.protoCodec.Unmarshal+0x118 /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/codec.go:85
# 0x11c34f6 Masami/backEnd/vendor/google.golang.org/grpc.(*protoCodec).Unmarshal+0x76 :35
# 0x11b21be Masami/backEnd/vendor/google.golang.org/grpc.recv+0x16e /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/rpc_util.go:382
# 0x11bcce1 Masami/backEnd/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xf1 /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/stream.go:398
# 0x11cc441 Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer.(*eventsChatClient).Recv+0x61 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric/protos/peer/events.pb.go:559
# 0x1665d0b Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events
###协程情况：
goroutine profile: total 4023
3964 @ 0x4382fa 0x4383de 0x40f59d 0x40f2ad 0x17d90cc 0x16729b7 0x465691
# 0x17d90cb Masami/backEnd/vendor/ILIOS/core/blockchain.(*BlockchainChannel).RegisterBlockEvent.func1+0x1cb /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491
# 0x16729b6 Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events.(*EventHub).Recv.func1+0x336 /opt/gopath/src/Masami/backEnd/vendor/github.com/hyperledger/fabric-sdk-go/pkg/fabric-client/events/eventhub.go:280
4 @ 0x4382fa 0x433417 0x432a59 0x539888 0x5398f4 0x53b047 0x54ee20 0x643e08 0x644374 0x64807d 0x57dcf2 0x47aba9 0x47ad18 0xafc78b 0xafd0e4 0x119c18f 0x118eb7d 0x465691
# 0x432a58 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x539887 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x5398f3 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x53b046 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x54ee1f net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x643e07 crypto/tls.(*block).readFromUntil+0x97 /usr/local/go/src/crypto/tls/conn.go:488
# 0x644373 crypto/tls.(*Conn).readRecord+0xc3 /usr/local/go/src/crypto/tls/conn.go:590
# 0x64807c crypto/tls.(*Conn).Read+0x11c /usr/local/go/src/crypto/tls/conn.go:1134
# 0x57dcf1 bufio.(*Reader).Read+0x311 /usr/local/go/src/bufio/bufio.go:213
# 0x47aba8 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x47ad17 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0xafc78a Masami/backEnd/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:237
# 0xafd0e3 Masami/backEnd/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /opt/gopath/src/Masami/backEnd/vendor/golang.org/x/net/http2/frame.go:492
# 0x119c18e Masami/backEnd/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http_util.go:608
# 0x118eb7c Masami/backEnd/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xcc /opt/gopath/src/Masami/backEnd/vendor/google.golang.org/grpc/transport/http2_client.go:1115
###内存消耗图：
(pprof) top
31857.57kB of 32881.59kB total (96.89%)
Dropped 1095 nodes (cum <= 164.41kB)
Showing top 10 nodes out of 112 (cum >= 512.02kB)
flat flat% sum% cum cum%
22576.68kB 68.66% 68.66% 22576.68kB 68.66% Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes
2048.81kB 6.23% 74.89% 2048.81kB 6.23% runtime.malg
2048.12kB 6.23% 81.12% 24112.79kB 73.33% Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).dec_slice_slice_byte
1536.08kB 4.67% 85.79% 1536.08kB 4.67% reflect.unsafe_New
1078.31kB 3.28% 89.07% 1078.31kB 3.28% runtime.makemap
520.04kB 1.58% 90.65% 520.04kB 1.58% Masami/backEnd/vendor/github.com/go-sql-driver/mysql.(*buffer).fill
513.31kB 1.56% 92.21% 513.31kB 1.56% regexp/syntax.(*compiler).compile
512.16kB 1.56% 93.77% 512.16kB 1.56% Masami/backEnd/vendor/golang.org/x/net/trace.(*histogram).Add
512.02kB 1.56% 95.33% 512.02kB 1.56% Masami/backEnd/vendor/google.golang.org/grpc.newClientStream
512.02kB 1.56% 96.89% 512.02kB 1.56% runtime.rawstringtmp
分析上述信息可知：
- Masami/backEnd/vendor/github.com/golang/protobuf/proto.(*Buffer).DecodeRawBytes消耗了大部分内存
- /opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491处有协程泄漏
#原因分析：
- 一般来讲，有协程泄漏的地方肯定会导致内存泄漏，之前遇到过grpc接收区块时，资源不释放，总之我们首先解决协程泄漏的问题。
##review代码：
/opt/gopath/src/Masami/backEnd/vendor/ILIOS/core/blockchain/blockchainChannel.go:1491:
//RegisterBlockEvent 通用的block监听方法，不会过滤channelid
func (channel *BlockchainChannel) RegisterBlockEvent(peername string, filterByChannelID bool) (chan *common.Block, error) {
logs.Debug("gggRegisterBlockEvent\n")
client, _, err := setupClient(channel.UserId, peername)
if err != nil {
return nil, err
}
eventHub, err := events.NewEventHub(client)
if err != nil {
return nil, err
}
peerinfo, err := getPeerKeystorePathAndUrl(channel.UserId, peername)
if err != nil {
return nil, err
}
eventHub.SetPeerAddr(peerinfo.EventUrl, string(peerinfo.TLSCert.Content), peerinfo.CommonName)
err = eventHub.Connect()
if err != nil {
return nil, err
}
blockchan := make(chan *common.Block)
eventHub.RegisterBlockEvent(func(block *common.Block) {
logs.Info("ggggReceived callback on block event")
if filterByChannelID == false {
blockchan <- block
logs.Debug("ggggfilterByChannelID\n")
} else {
for _, tdata := range block.Data.Data {
channelid, err := getChannelID(tdata)
if err == nil && channelid == channel.ChannelID {
blockchan <- block
logs.Debug("gggggChannelID:%s\n", channelid)
break
}
}
}
})
return blockchan, nil
}
func GetMonitor(namespace string, channel string, peer string) {
logs.Debug("ggggGetMonitor\n")
for {
select {
case block := <-common.BlockEventhub[namespace][channel][peer].Block:
logs.Debug("AAA channel:%s,%s,%s\n", namespace, channel, peer)
blocknum, transnum, ectivetransnum, nowtime := DesBlockMonitor(block)
logs.Debug("BBB channel: ", channel, "peer: ", peer, "blocknum: ", blocknum, "transnum: ", transnum, "ectivetransnum: ", ectivetransnum, "nowtime: ", nowtime)
// logs.Debug("AAA nowBlowNum: ", common.BlockEventhub[namespace][channel][peer].BlockNum)
if blocknum > common.BlockEventhub[namespace][channel][peer].BlockNum {
common.BlockEventhub[namespace][channel][peer].BlockNum = blocknum
common.UserChannelTps[namespace][channel].ChannelLock.Lock()
index := len(common.UserChannelTps[namespace][channel].TPSTimeStamp) - 1
common.UserChannelTps[namespace][channel].TPSList[index] += transnum
common.UserChannelTps[namespace][channel].BlockMakeList[index] += 1
common.UserChannelTps[namespace][channel].ChannelLock.Unlock()
}
}
}
}
func RegisterEventInit(namespace string, channelname string, peername []string) {
if _, ok := common.BlockEventhub[namespace]; !ok {
common.BlockEventhub[namespace] = make(map[string](map[string]*common.ChannelMonitor))
}
if _, ok := common.BlockEventhub[namespace][channelname]; !ok {
common.BlockEventhub[namespace][channelname] = make(map[string]*common.ChannelMonitor)
}
logs.Debug("gggggpeername:%s\n", peername)
for i, peer := range peername {
logs.Debug("ggggi=%d,peer:%s\n", i, peer)
go func() {
logs.Debug("ggggnamespace: ", namespace, channelname, peer)
address, err := GetPeerEventAddress(namespace, peer)
if err != nil {
logs.Error("RegisterEventInit GetPeerEventAddress Error: ", err)
return
}
showlog := false
ticker := time.NewTicker(2 * time.Second)
registerflag := false
for _ = range ticker.C {
if err := tcpTimeout(address); err != nil {
logs.Debug("ggggtcptimeout,addr:%s, err:%s\n", address, err)
showlog = false
logs.Warning("*************EVENT TRY TO CONNECT user %s %v***********", namespace, err)
channel := blockchain.NewBlockchainChannel(namespace, channelname)
blockchan, err := channel.RegisterBlockEvent(peer, true)
if err != nil {
logs.Error("RegisterEventInit Error: ", err)
} else {
common.BlockEventhubLock.Lock()
if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
go GetMonitor(namespace, channelname, peer)
}
common.BlockEventhubLock.Unlock()
}
} else {
if !registerflag {
registerflag = true
channel := blockchain.NewBlockchainChannel(namespace, channelname)
blockchan, _ := channel.RegisterBlockEvent(peer, true)
if err != nil {
logs.Error("RegisterEventInit Error: ", err)
} else {
logs.Debug("gggggregblockevent\n")
common.BlockEventhubLock.Lock()
if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
go GetMonitor(namespace, channelname, peer)
}
common.BlockEventhubLock.Unlock()
}
}
if showlog == false {
logs.Debug("*************EVENT HAS CONNECTED user %s ***********", namespace)
showlog = true
}
}
}
}()
}
}
func GetBlockFromFabricInit(namespace string) {
channels, _ := GetAllChannels(namespace)
for _, channel := range channels {
InitUserChannelTps(namespace, channel.ChannelName)
// InitUserChannelTpsNode(namespace, channel)
GetBlockFromFabricByChannel(namespace, channel)
// go CalTPS(namespace, channel)
//register eventhub
if channel.UserNodename != "" && channel.Status == common.CHANNELJOIN {
comp, err := GetNameSpaceOrgNameByNamespace(namespace)
if err != nil {
logs.Error("Get company by namespace error: ", err)
continue
}
nodes, err := GetPeerByNamespaceAndChannelAndComp(namespace, channel.ChannelName, comp)
if err != nil {
logs.Error("GetPeerByNamespaceAndChannelAndComp error: ", err)
continue
}
nodeStrs := []string{}
for _, node := range nodes {
nodeStrs = append(nodeStrs, node.Name)
}
logs.Debug("ggggGetBlockFromFabricInitRegisterEventInit:%s,%s,%s\n", namespace, channel.ChannelName, nodeStrs)
RegisterEventInit(namespace, channel.ChannelName, nodeStrs)
//ADD
go QueryChaincodeListInvoke(namespace, channel.ChannelName)
}
}
}
##可能原因：
- blockchan被阻塞：
- getmonitor接收的不完全，接收区块处理过程可能被锁住（注释掉处理区块的操作，现象不变，所以排除该因）
- 还有blockchain通道的block没有被接收
- 有全局变量导致协程无法释放（协程可以到达return，应该也不会由此引起）
- 协程没有返回，一直运行（通过加日志发现是可以到达return的，所以排除该因）
于是加了一些打印，查看可能出现问题的地方
##日志信息：
2018/06/07 10:52:10 [I] [eventhub.go:280] ggggReceived callback on block event
2018/06/07 10:52:10 [D] [eventhub.go:280] gggggChannelID:mychannel
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] after send chan
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] before return
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] AAA channel:4ddc1373-e73b-4679-8560-c3d26b4c64ff,mychannel,peer-1-yunphant1
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] BBB channel: mychannel peer: peer-1-yunphant1 blocknum: 84920 transnum: 1 ectivetransnum: 1 nowtime: 1528368730
2018/06/07 10:52:10 [D] [asm_amd64.s:2197] recv count2, time:2018-06-07 10:52:10.560654537 +0000 UTC, Recv blockEvent:&{header:<6:73 0:"\x01K\xb8" } 4:11890 157618:/* unknown wire type 6 */ 145:40 747:/* unexpected EOF
2018/06/07 10:52:08 [D] [browserbass.go:43] gggggpeername:[peer-0-yunphant1 peer-1-yunphant1]
2018/06/07 10:52:08 [D] [browserbass.go:43] ggggi=0,peer:peer-0-yunphant1
2018/06/07 10:52:08 [D] [browserbass.go:43] ggggi=1,peer:peer-1-yunphant1
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ================QueryBaasChaincodeEvent BEGIN=================
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ggggnamespace: 4ddc1373-e73b-4679-8560-c3d26b4c64ff mychannel peer-1-yunphant1
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ============QueryChaincodeListInvoke Begin=========== mychannel
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ====== Begin QueryChannelListBaaS ======
2018/06/07 10:52:08 [D] [asm_amd64.s:2197] ggggnamespace: 4ddc1373-e73b-4679-8560-c3d26b4c64ff mychannel peer-1-yunphant1
##现象总结：
- 每两秒产生一个协程
- 接收区块的协程也就是getmonitor函数，每两秒才接收一个
- grpc.Recv()要比每两秒一个区块多
- 创建blockchan时，两个peername居然一样
if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
go GetMonitor(namespace, channelname, peer)
}
这里会对同样namespace,peername的map进行判断，第二次时不会对
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
重新赋值，导致blockchan的指针地址不一致，也没有起GetMonitor()，第一次创建的getmonitor没法接受第二次创建的blockchan里的block(因为blockchan是不一致的) ,所以会造成blockchan阻塞。
其实这段代码逻辑上也是有风险的，最好这样写，如果相同namesapce和peer,就不要去registorBlockEvent：
if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
//registBlockEvent的操作也应该放在这个判断里面，不然造成前后不一致，导致内存泄漏，系统崩溃的问题
channel := blockchain.NewBlockchainChannel(namespace, channelname)
blockchan, _ := channel.RegisterBlockEvent(peer, true)
common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
go GetMonitor(namespace, channelname, peer)
}
#解决方案：
- 闭包函数，采用传参方式
- registorBlockevent操作放在判断里面
for i, peer := range peername {
logs.Debug("ggggi=%d,peer:%s\n", i, peer)
...
go func(peer string) {
if _, ok := common.BlockEventhub[namespace][channelname][peer]; !ok {
//registBlockEvent的操作也应该放在这个判断里面，不然造成前后不一致，导致内存泄漏，系统崩溃的问题
channel := blockchain.NewBlockchainChannel(namespace, channelname)
blockchan, _ := channel.RegisterBlockEvent(peer, true)
common.BlockEventhub[namespace][channelname][peer] = new(common.ChannelMonitor)
common.BlockEventhub[namespace][channelname][peer].Block = blockchan
go GetMonitor(namespace, channelname, peer)
}
...
}(peer)
像for i,peer := range peername这种操作，因为golang所有的goroutine都会并发执行。for range循环也许在第一个最多第二个goroutine还在运行的时候就运行完了，peer变量只会有最后一次循环时候的值。也就是说即使不是全部的goroutine也是大部分的goroutine会处理这些变量的相同的值，导致和实际预期不一致的结果。
不过，我们可以声明可以接收参数的匿名函数，来解决这些类型的闭包问题。
到此，也解释了之前比较疑惑的问题，两秒泄漏一个协程，getMonitor（）每两秒接收一个区块，但是还会有blockchan阻塞的现象。因为两个节点中只有最后一个peer的blockchan没有被接收，event Recv函数里，会接收到注册的event，第二个peer的block被阻塞了，也就有了每两秒产生一个goroutine了，因为区块时每两秒产生一个，第二个peer的eventblock注册了两次，会接收两次。
添加以上解决方案的话，保证了每个通道的block都可被接收，不会有协程阻塞。泄漏问题也得以解决，内存稳定。之前decodeRawBytes占了大部分内存的问题已经没有，下面是内存消耗的情况：
PS:
查看进程内存情况
a、top
b、pmap -d 12345
c、ps -e -o ‘pid,comm,args,pcpu,rsz,vsz,stime,user,uid’ 其中rsz是是实际内存
ps -e -o ‘pid,comm,args,pcpu,rsz,vsz,stime,user,uid’ | grep backEnd | sort -nrk5
##golang闭包函数
闭包的字面定义：闭包是由函数及其相关引用环境组合而成的实体(即：闭包=函数+引用环境)。闭包函数可以直接引用外层代码定义的变量，闭包函数里面引用的是变量的地址，也就是说闭包函数会改变外面变量的值，当goroutine被调度时，改地址的值才会被传递给goroutine 函数。
for i, peer := range peername {
logs.Debug("ggggi=%d,peer:%s\n", i, peer)
go func(peer string) {
dosomething(peer)
}(peer)
当这种for i,peer := range peername的程序时需要注意，如果想传入遍历peername的peer的值的话需要通过接受参数的形式;
如果代码：
for i, peer := range peername {
logs.Debug("ggggi=%d,peer:%s\n", i, peer)
go func() {
dosomething(peer)
}()
那么peer的基本上是peername里的最后一个值，因为go都是并发执行，可能第一个go func()还没运行完，range peername就运行完了，那么后面的go func(){}的peer值就是最后一个。