glusterfs mount client crash

1. debug glusterfs

2. enable coredump

• 1.change /etc/security/limits.conf

*               soft    core             unlimited

• 2. set coredump dir

# echo "/data/coredump/core.%e.%p" > /proc/sys/kernel/core_pattern

• 3. set bashrc

ulimit -c LIMIT

3. execute mnt.sh

• mnt.sh

fusermount -uz /data/glusterfs_cv
mount -t glusterfs -o acl 172.25.78.19:/train_vol /data/glusterfs_cv
ps -ef|grep fuse|grep glusterfs

- output
[root@szdpl1487 ~]# ./mnt.sh 
root     123165      1  0 10:06 ?        00:00:07 /usr/local/sbin/glusterfs --acl --process-name fuse --volfile-server=172.25.78.19 --volfile-id=/train_vol /data/glusterfs_cv
root     154701      1  0 10:28 ?        00:00:00 /usr/local/sbin/glusterfs --acl --process-name fuse --volfile-server=172.25.78.19 --volfile-id=/train_vol /data/glusterfs_cv

4.gdb attach child process

[root@szdpl1487 ~]# gdb /usr/local/sbin/glusterfs
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-115.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later 
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
...
Reading symbols from /usr/local/sbin/glusterfsd...done.
(gdb) attach 123165

5.got calltrace from coredump

[root@ceph-10-193-27-15:/data/coredump]
# gdb /usr/local/sbin/glusterfs -c core.glfs_iotwr000.2308337
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later 
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
...
Reading symbols from /usr/local/sbin/glusterfsd...done.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `/usr/local/sbin/glusterfs --process-name fuse --volfile-server=10.193.26.1 --vo'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f7c0937ed14 in __GI__IO_default_xsputn () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install glibc-2.17-260.el7_6.3.x86_64 keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-37.el7_6.x86_64 libcom_err-1.42.9-13.el7.x86_64 libgcc-4.8.5-36.el7.x86_64 libselinux-2.5-14.1.el7.x86_64 libuuid-2.23.2-61.el7.x86_64 openssl-libs-1.0.2k-19.el7.x86_64 pcre-8.32-17.el7.x86_64 zlib-1.2.7-18.el7.x86_64
(gdb) bt
#0  0x00007f7c0937ed14 in __GI__IO_default_xsputn () from /lib64/libc.so.6
#1  0x00007f7c09371ea5 in __GI__IO_padn () from /lib64/libc.so.6
#2  0x00007f7c0934ccd5 in vfprintf () from /lib64/libc.so.6
#3  0x00007f7c09379f39 in vsnprintf () from /lib64/libc.so.6
#4  0x00007f7c093553c2 in snprintf () from /lib64/libc.so.6
#5  0x00007f7c0acfb25a in gf_log_glusterlog (ctx=0x9c0010, domain=domain@entry=0x7f7bf005eef0 "speech_vol-dht", file=file@entry=0x7f7bfc11f848 "dht-common.c", 
    function=function@entry=0x7f7bfc123a40 <__FUNCTION__.24346> "dht_readdirp_cbk", line=line@entry=6655, level=level@entry=GF_LOG_DEBUG, errnum=0, msgid=0, 
    appmsgstr=0x7f7c0b014b38, callstr=0x0, tv=..., graph_id=0, fmt=gf_logformat_withmsgid) at logging.c:1397
#6  0x00007f7c0acfb681 in gf_log_print_plain_fmt (ctx=ctx@entry=0x9c0010, domain=domain@entry=0x7f7bf005eef0 "speech_vol-dht", 
    file=file@entry=0x7f7bfc11f848 "dht-common.c", function=function@entry=0x7f7bfc123a40 <__FUNCTION__.24346> "dht_readdirp_cbk", line=line@entry=6655, 
    level=level@entry=GF_LOG_DEBUG, errnum=, errnum@entry=0, msgid=, msgid@entry=0, appmsgstr=, 
    appmsgstr@entry=0x7f7c0b014b38, callstr=, tv=..., graph_id=, fmt=) at logging.c:1709
#7  0x00007f7c0acfa5b5 in _gf_msg_internal (graph_id=0, callstr=, appmsgstr=0x7f7c0b014b38, msgid=0, errnum=0, level=GF_LOG_DEBUG, line=6655, 
    function=0x7f7bfc123a40 <__FUNCTION__.24346> "dht_readdirp_cbk", file=0x7f7bfc11f848 "dht-common.c", domain=0x7f7bf005eef0 "speech_vol-dht") at logging.c:2049
#8  _gf_msg (domain=0x7f7bf005eef0 "speech_vol-dht", file=file@entry=0x7f7bfc11f848 "dht-common.c", 
    function=function@entry=0x7f7bfc123a40 <__FUNCTION__.24346> "dht_readdirp_cbk", line=line@entry=6655, level=level@entry=GF_LOG_DEBUG, errnum=errnum@entry=0, 
    trace=trace@entry=0, msgid=msgid@entry=0, fmt=fmt@entry=0x7f7bfc11f9c4 "Processing entries from %s") at logging.c:2124
#9  0x00007f7bfc0ef699 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=0x7f7bf0082c90, this=0x7f7bf0094f60, op_ret=op_ret@entry=17, op_errno=0, 
    orig_entries=orig_entries@entry=0x7f7c0b015e70, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6655
#10 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b58ac34e8, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3587541, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#11 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b016190, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#12 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b59947c18, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3584277, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#13 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b0164b0, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#14 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b59947b08, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3581013, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#15 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b0167d0, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#16 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b599479f8, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3577749, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#17 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b016af0, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#18 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b599478e8, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3574485, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#19 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b016e10, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#20 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b599477d8, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3570965, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
#21 0x00007f7bfc0f00d9 in dht_readdirp_cbk (frame=frame@entry=0x7f7b58003b28, cookie=, this=, op_ret=, op_ret@entry=17, 
    op_errno=0, orig_entries=orig_entries@entry=0x7f7c0b017130, xdata=xdata@entry=0x7f7b5800ca28) at dht-common.c:6854
#22 0x00007f7bfc347c9f in rda_readdirp (frame=frame@entry=0x7f7b599476c8, this=this@entry=0x7f7bf0082c90, fd=0x7f7b5c001b38, size=, 
    off=off@entry=3567701, xdata=0x7f7b5800ca28) at readdir-ahead.c:460
---Type  to continue, or q  to quit---