使用ebpf 监控golang 应用
一、背景
使用ebpf 监控grpc-go的应用，grpc-go http2 client的处理点
func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.HeaderField, onWrite func()) error {
......
}
// operateHeaders takes action on the decoded headers.
func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) {
}使用ebpf 监控埋点:
SEC("uprobe/google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
int uprobe__probe_http2_client_operate_headers(struct pt_regs* ctx) {
}
运行流程:
我们在 uprobe__probe_http2_client_operate_headers 里做一些处理，统计grpc的数据，进行遥测。编写代码过程中我们需要关注的一些事情
1、golang 程序中 t *http2Client 和 frame *http2.MetaHeadersFrame 在寄存器中的位置是如何分配的？
type MetaHeadersFrame struct {
*HeadersFrame
// Fields are the fields contained in the HEADERS and
// CONTINUATION frames. The underlying slice is owned by the
// Framer and must not be retained after the next call to
// ReadFrame.
//
// Fields are guaranteed to be in the correct http2 order and
// not have unknown pseudo header fields or invalid header
// field names or values. Required pseudo header fields may be
// missing, however. Use the MetaHeadersFrame.Pseudo accessor
// method access pseudo headers.
Fields []hpack.HeaderField
// Truncated is whether the max header list size limit was hit
// and Fields is incomplete. The hpack decoder state is still
// valid, however.
Truncated bool
}2、http2Client 和 MetaHeadersFrame 结构体中各个成员的偏移量如何确定？
type http2Client struct {
lastRead int64 // Keep this field 64-bit aligned. Accessed atomically.
ctx context.Context
cancel context.CancelFunc
ctxDone <-chan struct{} // Cache the ctx.Done() chan.
userAgent string
// address contains the resolver returned address for this transport.
// If the `ServerName` field is set, it takes precedence over `CallHdr.Host`
// passed to `NewStream`, when determining the :authority header.
address resolver.Address
md metadata.MD
.....
}以 MetaHeadersFrame 为例子
我们在开发中需要找到 正确的offset 才能正确的取到内存。
二、我们需要做什么？
1、如何确定参数在寄存器中的位置？
我们使用llvm-dwarfdump-14 查看二进制中dwarf 构成:
llvm-dwarfdump-14 grpc-client |grep "loopyWriter).writeHeader" -C 100返回格式:
0x0018c375: DW_TAG_subprogram
DW_AT_name ("google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
DW_AT_low_pc (0x00000000007139a0)
DW_AT_high_pc (0x0000000000713df2)
DW_AT_frame_base (DW_OP_call_frame_cfa)
DW_AT_decl_file ("/home/zhanglei/data/grpc-demo/vendor/google.golang.org/grpc/internal/transport/controlbuf.go")
DW_AT_external (0x01)
0x0018c3d2: DW_TAG_formal_parameter
DW_AT_name ("l")
DW_AT_variable_parameter (0x00)
DW_AT_decl_line (677)
DW_AT_type (0x00000000000e6bc2 "google.golang.org/grpc/internal/transport.loopyWriter *")
DW_AT_location (0x001a603f:
[0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX
[0x00000000007139fc, 0x0000000000713df2): DW_OP_call_frame_cfa)
0x0018c3e0: DW_TAG_formal_parameter
DW_AT_name ("streamID")
DW_AT_variable_parameter (0x00)
DW_AT_decl_line (677)
DW_AT_type (0x0000000000082ad4 "uint32")
DW_AT_location (0x001a6085:
[0x00000000007139a0, 0x0000000000713a02): DW_OP_reg3 RBX
[0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg +8)
0x0018c3f5: DW_TAG_formal_parameter
DW_AT_name ("endStream")
DW_AT_variable_parameter (0x00)
DW_AT_decl_line (677)
DW_AT_type (0x0000000000082b87 "bool")
DW_AT_location (0x001a60cc:
[0x00000000007139a0, 0x0000000000713a02): DW_OP_reg2 RCX
[0x0000000000713a02, 0x0000000000713df2): DW_OP_fbreg +12)
0x0018c40b: DW_TAG_formal_parameter
DW_AT_name ("hf")
DW_AT_variable_parameter (0x00)
DW_AT_decl_line (677)
DW_AT_type (0x00000000000e2826 "[]golang.org/x/net/http2/hpack.HeaderField")
DW_AT_location (0x001a6113:
[0x00000000007139a0, 0x0000000000713a02): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8
[0x0000000000713a02, 0x0000000000713a4e): DW_OP_fbreg +16, DW_OP_piece 0x8, DW_OP_fbreg +24, DW_OP_piece 0x8, DW_OP_piece 0x8
[0x0000000000713a4e, 0x0000000000713df2): DW_OP_piece 0x8, DW_OP_fbreg +24, DW_OP_piece 0x8, DW_OP_piece 0x8)
0x0018c41a: DW_TAG_formal_parameter
DW_AT_name ("onWrite")
DW_AT_variable_parameter (0x00)
DW_AT_decl_line (677)
DW_AT_type (0x00000000000844f2 "func()")
DW_AT_location (0x001a6184:
[0x00000000007139a0, 0x0000000000713a02): DW_OP_reg9 R9)
数据格式简介:
我们可以找到func (l *loopyWriter) writeHeader 的参数的所在寄存器的位置
DW_AT_name ("l") 的 [0x00000000007139a0, 0x00000000007139fc): DW_OP_reg0 RAX 在寄存器0 上
DW_AT_name ("streamID") : DW_OP_reg3 RBX 在寄存器3上
DW_AT_name ("endStream") DW_OP_reg2 RCX 在寄存器2上
DW_AT_name ("hf") 在寄存器5 上 DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_reg4 RSI, DW_OP_piece 0x8, DW_OP_reg8 R8, DW_OP_piece 0x8
DW_AT_name ("onWrite") DW_OP_reg9 R9 在寄存器9上2、如何定位结构体中的成员
0x000e6c07: DW_TAG_structure_type
DW_AT_name ("google.golang.org/grpc/internal/transport.loopyWriter")
DW_AT_byte_size (88)
DW_AT_GO_kind (0x19)
DW_AT_GO_runtime_type (0x00000000000b9a20)
0x000e6c48: DW_TAG_member
DW_AT_name ("side")
DW_AT_data_member_location (0)
DW_AT_type (0x00000000000e5e96 "google.golang.org/grpc/internal/transport.side")
DW_AT_GO_embedded_field (0x00)
0x000e6c54: DW_TAG_member
DW_AT_name ("cbuf")
DW_AT_data_member_location (8)
DW_AT_type (0x00000000000e681d "google.golang.org/grpc/internal/transport.controlBuffer *")
DW_AT_GO_embedded_field (0x00)
0x000e6c60: DW_TAG_member
DW_AT_name ("sendQuota")
DW_AT_data_member_location (16)
DW_AT_type (0x0000000000082ad4 "uint32")
DW_AT_GO_embedded_field (0x00)
0x000e6c71: DW_TAG_member
DW_AT_name ("oiws")
DW_AT_data_member_location (20)
DW_AT_type (0x0000000000082ad4 "uint32")
DW_AT_GO_embedded_field (0x00)
0x000e6c7d: DW_TAG_member
DW_AT_name ("estdStreams")
DW_AT_data_member_location (24)
DW_AT_type (0x00000000000e6d3c "map[uint32]*google.golang.org/grpc/internal/transport.outStream")
DW_AT_GO_embedded_field (0x00)
0x000e6c90: DW_TAG_member
DW_AT_name ("activeStreams")
DW_AT_data_member_location (32)
DW_AT_type (0x00000000000e6eaa "google.golang.org/grpc/internal/transport.outStreamList *")
DW_AT_GO_embedded_field (0x00)
0x000e6ca5: DW_TAG_member
DW_AT_name ("framer")
DW_AT_data_member_location (40)
DW_AT_type (0x00000000000e6f8a "google.golang.org/grpc/internal/transport.framer *")
DW_AT_GO_embedded_field (0x00)
以 loopyWriter 结构体为例子:
type loopyWriter struct {
side side
cbuf *controlBuffer
sendQuota uint32
oiws uint32 // outbound initial window size.
// estdStreams is map of all established streams that are not cleaned-up yet.
// On client-side, this is all streams whose headers were sent out.
// On server-side, this is all streams whose headers were received.
estdStreams map[uint32]*outStream // Established streams.
// activeStreams is a linked-list of all streams that have data to send and some
// stream-level flow control quota.
// Each of these streams internally have a list of data items(and perhaps trailers
// on the server-side) to be sent out.
activeStreams *outStreamList
framer *framer
hBuf *bytes.Buffer // The buffer for HPACK encoding.
hEnc *hpack.Encoder // HPACK encoder.
bdpEst *bdpEstimator
draining bool
// Side-specific handlers
ssGoAwayHandler func(*goAway) (bool, error)
}side 的偏移量是 0
cbuf 是 8
sendQuota 是 16
oiws 是 20
estdStreams 是 24
activeStreams 是 32
framer 是 40
3、如何确定接口类型地址
nm grpc-client|grep "TCPConn,net.Conn"
000000000093d280 R go.itab.*net.TCPConn,net.Conn发现对应的符号表地址:
nm grpc-server|grep "TCPConn,net.Conn"
000000000093bfc0 R go.itab.*net.TCPConn,net.Conn4、使用gdb 证明这些数据
(gdb) i args
l = 0xc00011c060
streamID = 1
endStream = false
hf = {array = 0xc00007e870, len = 2, cap = 2}
onWrite = {void (void)} 0x0
~r0 =
(gdb) i r
rax 0xc00011c060 824634884192
rbx 0x1 1
rcx 0x0 0
rdx 0xc0001b29c0 824635500992
rsi 0x2 2
rdi 0xc00007e870 824634239088
rbp 0xc000129ec8 0xc000129ec8
rsp 0xc000129e78 0xc000129e78
r8 0x2 2
r9 0xc000021f80 824633859968
r10 0xc00007e8c0 824634239168
r11 0x1 1
r12 0x1 1
r13 0xffffffffffffffff -1
r14 0xc00019d520 824635413792
r15 0x0 0
rip 0x7160c0 0x7160c0
eflags 0x246 [ PF ZF IF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
发现l 位于寄存器
l -> rax
streamID -> rbx
endStream -> rcx
hf -> rdi
接口类型:
(gdb) p (*l.framer.writer).conn
$8 = {tab = 0x93bfc0 , data = 0xc000010290} 发现对应的符号表地址:
nm grpc-server|grep "TCPConn,net.Conn"
000000000093bfc0 R go.itab.*net.TCPConn,net.Conn三、发现golang应用
uprober 包含了 多个rule
每个rule 又包含Selector和 Checker
Selector 用来根据版本选择对应的Hooker
Checker 用来检查 动态库或者二进制文件是否满足匹配规则
运行流程：
四、golang 注册ebpf 钩子
五.使用偏移量获取golang变量
定义 location 结构体:
typedef struct {
__s64 stack_offset; // 栈上的偏移量
__s64 _register; // 寄存器的偏移量
__u8 in_register; //是否在寄存器上
__u8 exists;
} location_t;
// golang 切片
typedef struct {
location_t ptr;
location_t len;
location_t cap;
} slice_location_t;
golang 参数变量在寄存器的布局:
// This function was adapted from https://github.com/go-delve/delve:
// - https://github.com/go-delve/delve/blob/cd9e6c02a6ca5f0d66c1f770ee10a0d8f4419333/pkg/proc/internal/ebpf/bpf/trace.bpf.c#L43
// which is licensed under MIT.
static __always_inline int read_register(struct pt_regs* ctx, int64_t regnum, void* dest) {
// This volatile temporary variable is need when building with clang-14,
// or the verifier will complain that we dereference a modified context
// pointer.
//
// What happened in this case, is that the compiler tried to be smart by
// incrementing the context pointer, before jumping to code that will
// copy the value pointed to by the new pointer to `dest`. The generated
// code looked like this:
//
// r1 += 40 // Increment the ptr
// goto +3 // goto __builtin_memcpy
//
// What the memcpy does is deference the resulting pointer to get the
// CPU register value (that’s where the bug was), then put it in the
// dest location:
//
// r1 = *(u64 *)(r1 + 0) // BUG: Get the register value.
// // This is the "modified context pointer"
// *(u64 *)(r3 + 0) = r1 // Put it in dest
//
// By incrementing the pointer before dereferencing it, the verifier no
// longer considering r1 to be a pointer to the context, but as a
// pointer to some random memory address (even though it is in the
// memory the range of the context struct).
//
// What we want the compiler to generate is something like this:
//
// // Switch branch:
// r1 = *(u64 *)(r1 + 40) // read value to tmp var
// goto +30 // goto *dest = tmp
//
// // *dest = tmp
// *(u64 *)(r3 + 0) = r1
//
// This volatile `tmp` variable makes the compiler generate the code above.
volatile u64 tmp = 0;
switch (regnum) {
case 0: // RAX
tmp = ctx->ax;
break;
case 1: // RDX
tmp = ctx->dx;
break;
case 2: // RCX
tmp = ctx->cx;
break;
case 3: // RBX
tmp = ctx->bx;
break;
case 4: // RSI
tmp = ctx->si;
break;
case 5: // RDI
tmp = ctx->di;
break;
case 6: // RBP
tmp = ctx->bp;
break;
case 7: // RSP
tmp = ctx->sp;
break;
case 8: // R8
tmp = ctx->r8;
break;
case 9: // R9
tmp = ctx->r9;
break;
case 10: // R10
tmp = ctx->r10;
break;
case 11: // R11
tmp = ctx->r11;
break;
case 12: // R12
tmp = ctx->r12;
break;
case 13: // R13
tmp = ctx->r13;
break;
case 14: // R14
tmp = ctx->r14;
break;
case 15: // R15
tmp = ctx->r15;
break;
default:
return 1;
}
*(u64*)dest = tmp;
return 0;
} 读取偏移量：
static __always_inline int read_location(struct pt_regs* ctx, location_t* loc, size_t size, void* dest) {
if (!loc->exists) {
return 0;
}
if (loc->in_register) {
if (size != REG_SIZE) {
return 1;
}
return read_register(ctx, loc->_register, dest);
} else {
return read_stack(ctx, loc->stack_offset, size, dest);
}
}static __always_inline int read_stack(struct pt_regs* ctx, int64_t stack_offset, size_t size, void* dest) {
// `ctx->sp` is correct for both x86_64 and ARM64
uintptr_t stack_pointer = (uintptr_t) ctx->sp;
uintptr_t address = stack_pointer + stack_offset;
return bpf_probe_read_user(dest, size, (void*) address);
}举一个例子:
SEC("uprobe/google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader")
int uprobe__probe_loopy_writer_write_header(struct pt_regs* ctx) {
return 0;
}从map读取偏移量:
uint32_t pid = bpf_get_current_pid_tgid() >> 32;
go_http2_symaddrs_t* symaddrs = bpf_map_lookup_elem(&http2_symaddrs_map, &pid);
if (symaddrs == NULL) {
log_trace("uprobe__probe_loopy_writer_write_header:writeHeader:symaddrs is NULL\n");
return 0;
}偏移量存储结构体:
typedef struct {
// ---- itable symbols ----
// io.Writer interface types.
__s64 http_http2bufferedWriter; // "go.itab.*net/http.http2bufferedWriter,io.Writer
__s64 transport_bufWriter; // "google.golang.org/grpc/internal/transport.bufWriter,io.Writer
// ---- function argument locations ----
// Arguments of net/http.(*http2Framer).WriteDataPadded.
location_t http2Framer_WriteDataPadded_f_loc; // 8
location_t http2Framer_WriteDataPadded_streamID_loc; // 16
location_t http2Framer_WriteDataPadded_endStream_loc; // 20
location_t http2Framer_WriteDataPadded_data_ptr_loc; // 24
location_t http2Framer_WriteDataPadded_data_len_loc; // 32
// Arguments of golang.org/x/net/http2.(*Framer).WriteDataPadded.
location_t http2_WriteDataPadded_f_loc; // 8
location_t http2_WriteDataPadded_streamID_loc; // 16
location_t http2_WriteDataPadded_endStream_loc; // 20
location_t http2_WriteDataPadded_data_ptr_loc; // 24
location_t http2_WriteDataPadded_data_len_loc; // 32
// Arguments of net/http.(*http2Framer).checkFrameOrder.
location_t http2Framer_checkFrameOrder_fr_loc; // 8
location_t http2Framer_checkFrameOrder_f_loc; // 16
// Arguments of golang.org/x/net/http2.(*Framer).checkFrameOrder.
location_t http2_checkFrameOrder_fr_loc; // 8
location_t http2_checkFrameOrder_f_loc; // 16
// Arguments of net/http.(*http2writeResHeaders).writeFrame.
location_t writeFrame_w_loc; // 8
location_t writeFrame_ctx_loc; // 16
// Arguments of golang.org/x/net/http2/hpack.(*Encoder).WriteField.
location_t WriteField_e_loc; // 8
// Note that the HeaderField `f` is further broken down to its name and value members.
// This is done so we can better control the location of these members from user-space.
// In theory, there could be an ABI that splits these two members across stack and registers.
location_t WriteField_f_name_loc; // 16
location_t WriteField_f_value_loc; // 32
// Arguments of net/http.(*http2serverConn).processHeaders.
location_t processHeaders_sc_loc; // 8
location_t processHeaders_f_loc; // 16
// Arguments of google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders.
location_t http2Server_operateHeaders_t_loc; // 8
location_t http2Server_operateHeaders_frame_loc; // 16
// Arguments of google.golang.org/grpc/internal/transport.(*http2Client).operateHeaders.
location_t http2Client_operateHeaders_t_loc; // 8
location_t http2Client_operateHeaders_frame_loc; // 16
// Arguments of google.golang.org/grpc/internal/transport.(*loopyWriter).writeHeader.
location_t writeHeader_l_loc; // 8
location_t writeHeader_streamID_loc; // 16
location_t writeHeader_endStream_loc; // 20
slice_location_t writeHeader_hf_loc; // 24
// ---- struct member offsets ----
// Struct member offsets.
// Naming maintains golang style: __offset
// Note: values in comments represent known offsets, in case we need to fall back.
// Eventually, they should be removed, because they are not reliable.
// Members of golang.org/x/net/http2/hpack.HeaderField.
int32_t HeaderField_Name_offset; // 0
int32_t HeaderField_Value_offset; // 16
// Members of google.golang.org/grpc/internal/transport.http2Server.
int32_t http2Server_conn_offset; // 16 or 24
// Members of google.golang.org/grpc/internal/transport.http2Client.
int32_t http2Client_conn_offset; // 64
// Members of google.golang.org/grpc/internal/transport.loopyWriter.
int32_t loopyWriter_framer_offset; // 40
// Members of golang.org/x/net/net/http2.Framer.
int32_t Framer_w_offset; // 112
// Members of golang.org/x/net/http2.MetaHeadersFrame.
int32_t MetaHeadersFrame_HeadersFrame_offset; // 0
int32_t MetaHeadersFrame_Fields_offset; // 0
// Members of golang.org/x/net/http2.HeadersFrame.
int32_t HeadersFrame_FrameHeader_offset; // 0
// Members of golang.org/x/net/http2.FrameHeader.
int32_t FrameHeader_Type_offset; // 1
int32_t FrameHeader_Flags_offset; // 2
int32_t FrameHeader_StreamID_offset; // 8
// Members of golang.org/x/net/http2.DataFrame.
int32_t DataFrame_data_offset; // 16
// Members of google.golang.org/grpc/internal/transport.bufWriter.
int32_t bufWriter_conn_offset; // 40
// Members of net/http.http2serverConn.
int32_t http2serverConn_conn_offset; // 16
int32_t http2serverConn_hpackEncoder_offset; // 360
// Members of net/http.http2HeadersFrame
int32_t http2HeadersFrame_http2FrameHeader_offset; // 0
// Members of net/http.http2FrameHeader.
int32_t http2FrameHeader_Type_offset; // 1
int32_t http2FrameHeader_Flags_offset; // 2
int32_t http2FrameHeader_StreamID_offset; // 8
// Members of golang.org/x/net/http2.DataFrame.
int32_t http2DataFrame_data_offset; // 16
// Members of net/http.http2writeResHeaders.
int32_t http2writeResHeaders_streamID_offset; // 0
int32_t http2writeResHeaders_endStream_offset; // 48
// Members of net/http.http2MetaHeadersFrame.
int32_t http2MetaHeadersFrame_http2HeadersFrame_offset; // 0
int32_t http2MetaHeadersFrame_Fields_offset; // 8
// Members of net/http.http2Framer.
int32_t http2Framer_w_offset; // 112
// Members of net/http.http2bufferedWriter
int32_t http2bufferedWriter_w_offset; // 0
} go_http2_symaddrs_t; 读取偏移量:
void* loopy_writer_ptr = NULL;
if (read_location(ctx, &symaddrs->writeHeader_l_loc,
sizeof(loopy_writer_ptr), &loopy_writer_ptr)) {
log_trace("uprobe__probe_loopy_writer_write_header:5\n");
return 0;
}
六、如何记录错误调试
1、打日志调试
#define log_trace(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
})
/* Macro to output debug logs to /sys/kernel/debug/tracing/trace_pipe
*/
#if DEBUG == 1
#define log_debug(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
})
#else使用:
log_debug("[grpc-c:lookup_version:]grpc-c version is not support;pid:%d;version:%d;\n", pid, *version);
2、记录metrics
主要记录行号和文件
#ifndef TRACER_TELEMETRY
#define TRACER_TELEMETRY
#ifndef TRACER_TELEMETRY_KEY_LIMIT
#define TRACER_TELEMETRY_KEY_LIMIT 4096
#endif
typedef struct {
char file[TRACER_TELEMETRY_KEY_LIMIT];
uint64_t line;
} tracer_telemetry_key;
BPF_HASH_MAP(tracer_telemetry, tracer_telemetry_key, uint64_t, 1024);
BPF_PERCPU_ARRAY_MAP(tracer_telemetry_heap, __u32, tracer_telemetry_key, 1);
static __always_inline tracer_telemetry_key* alloc_tracer_telemetry_key() {
uint32_t kZero = 0;
tracer_telemetry_key* value = bpf_map_lookup_elem(&tracer_telemetry_heap, &kZero);
if (value == NULL) {
return NULL;
}
value->line = 0;
return value;
}
static __always_inline void increment_tracer_telemetry_count(const char* file, uint64_t line) {
tracer_telemetry_key* key = alloc_tracer_telemetry_key();
if (key == NULL) {
return;
}
if (bpf_probe_read_str(&key->file, sizeof(key->file), file) == -1) {
return;
}
key->line = line;
uint64_t *val = NULL;
val = bpf_map_lookup_elem(&tracer_telemetry, key);
if (val == NULL) {
log_debug("tracer:key->file:%s;file:%s\n", key->file, file);
uint64_t tmp_value = 0;
bpf_map_update_with_telemetry(tracer_telemetry, key, &tmp_value, BPF_NOEXIST);
return;
}
val = bpf_map_lookup_elem(&tracer_telemetry, key);
if (val == NULL) {
return;
}
__sync_fetch_and_add(val, 1);
}
#ifndef INCR_TRACER_COUNT
#define INCR_TRACER_COUNT increment_tracer_telemetry_count(__FILE__, __LINE__)
#endif
#endif