kernel crash analysis
1. write one self trigger ramdump method
#echo c > /proc/sysrq-trigger
2. load kernel dump file to crash analyzer
crash vmlinux /home/xxxxx/debug/K1939EL.RAM
or
crash -m phys_base=offset(0x80000000) vmlinux K0342EL.RAM
3.setup crash environment
mkdir debug
cd debug
wget http://people.redhat.com/anderson/crash-7.0.0.tar.gz
tar -zxvf crash-7.0.0.tar.gz
cd crash-7.0.0
make target=ARM
sudo make install crash
sudo make extensions ( and it will generate some extension lib)
4. some commands
a. bt pid | task--- display stack backtrace
bt
PID: 0 TASK: c06913a8 CPU: 0 COMMAND: "swapper/0"
#0 [<c0489da4>] (__schedule) from [<c048a47c>]
#1 [<c048a3fc>] (schedule) from [<c048a6d4>]
#2 [<c048a6b0>] (schedule_preempt_disabled) from [<c0010a48>]
#3 [<c0010974>] (cpu_idle) from [<c04795d4>]
#4 [<c0479564>] (rest_init) from [<c06397c4>]
#5 [<c063951c>] (start_kernel) from [<8000803c>]
b. log or log > 1.log to get log
c. objdump to generate asm code from elf
./../../../../prebuilt/linux-x86/toolchain/arm-eabi-4.4.3/bin/arm-eabi-objdump -D vmlinux > kernel.asm
d.read data from the memory
#rd
e.# irq
f. ps to list all threads,
g. task to list all task information
h. help to get all manual information
5. example.
find corrupt, and we will analyze pc(r15), sp(r13), fp(r11).
[ 250.553810:0] Internal error: Oops: 805 [#1] PREEMPT SMP ARM
[ 250.559479:0] Modules linked in: bcmdhd mali ump
[ 250.564216:0] CPU: 0 Tainted: G W (3.4.5-g3d90f8c-dirty #27)
[ 250.571118:0] PC is at sysrq_handle_crash+0x20/0x2c
[ 250.576086:0] LR is at __handle_sysrq+0xa8/0x154
[ 250.580716:0] pc : [<c01f0600>] lr : [<c01f0c38>] psr: 60000093
[ 250.580725:0] sp : e5347ec0 ip : e5347ed0 fp : e5347ecc
[ 250.592650:0] r10: e5347f78 r9 : ee98d10c r8 : 00000000
[ 250.598139:0] r7 : 60000013 r6 : 00000063 r5 : 00000007 r4 : c06ab6f0
[ 250.604851:0] r3 : 00000001 r2 : 00000000 r1 : 60000093 r0 : 00000063
[ 250.611645:0] Flags: nZCv IRQs off FIQs on Mode SVC_32 ISA ARM Segment user
[ 250.619135:0] Control: 10c53c7d Table: 9e09406a DAC: 00000015
[ 250.625064:0]
[ 250.625069:0] PC: 0xc01f0580:
[ 250.629771:0] 0580 1a000001 eb0a6ca0 ea000006 eb0a6c9e e1a00007 e1a01006 ebffe0a0 ea000001
[ 250.638194:0] 05a0 e1a00006 ebffdb8f e1a00004 e89da9f8 c07143ec c06ab444 e1a0c00d e92dd800
[ 250.646699:0] 05c0 e24cb004 e59f3010 e5932000 e3520000 05d30004 13a00001 e89da800 c0689ee0
[ 250.655207:0] 05e0 e1a0c00d e92dd800 e24cb004 e59f2014 e3a03001 e5823000 f57ff04f e3a02000
[ 250.663643:0] 0600 e5c23000 e89da800 c06e5dac e1a0c00d e92dd830 e24cb004 e59f401c e2405030
[ 250.672072:0] 0620 e59f0018 e3a03007 e1a01005 e5843000 eb0a4e54 e5845000 e89da830 c06a00b0
[ 250.680576:0] 0640 c05b8641 e1a0c00d e92dd830 e24cb004 e5904000 e1a05000 eb01caf8 e2840024
[ 250.689083:0] 0660 ebf96722 e1a00005 eb01cab3 e1a00004 ebfb3c89 e89da830 e1a0c00d e92dd878
[ 250.697522:0]
[ 250.697527:0] LR: 0xc01f0bb8:
[ 250.702228:0] 0bb8 e5832000 e1a07000 e59f00fc eb0a4cef e1a00006 ebffffaf e2504000 0a00001b
[ 250.710732:0] 0bd8 e3580000 0a00000d e59f30e0 e594200c e5d31004 e3510000 13a03001 1a000005
[ 250.719168:0] 0bf8 e5933000 e3530001 0a000002 e1130002 03a03000 13a03001 e31300ff 0a000008
[ 250.727664:0] 0c18 e5941008 e59f00a8 eb0a4cd8 e59f3094 e1a00006 e5835000 e5943000 e12fff33
[ 250.736165:0] 0c38 ea00001b e59f008c eb0a4cd0 ea000018 e59f0084 eb0a4ccd e59f6080 e5b63004
[ 250.744597:0] 0c58 e3530000 159f1074 13a02000 1a000001 ea000008 e2822001 e5b10004 e1530000
[ 250.753019:0] 0c78 1afffffb e1520004 1a000002 e59f0050 e5931004 eb0a4cbd e2844001 e3540024
[ 250.761523:0] 0c98 1affffed e59f003c eb0a4cb8 e59f3014 e5835000 e59f0008 e1a01007 eb0a68fa
[ 250.770027:0]
[ 250.770031:0] SP: 0xe5347e40:
[ 250.774655:0] 7e40 00000004 205b0022 30353220 3533352e 3a373130 00205d30 c01f0600 60000093
[ 250.783156:0] 7e60 ffffffff e5347eac e5347ecc e5347e78 c000ef98 c0009174 00000063 60000093
[ 250.791662:0] 7e80 00000000 00000001 c06ab6f0 00000007 00000063 60000013 00000000 ee98d10c
[ 250.800091:0] 7ea0 e5347f78 e5347ecc e5347ed0 e5347ec0 c01f0c38 c01f0600 60000093 ffffffff
[ 250.808599:0] 7ec0 e5347ef4 e5347ed0 c01f0c38 c01f05ec e5347f78 00000002 c01f0ce4 de0ef180
[ 250.817105:0] 7ee0 00000002 b7a5dd34 e5347f0c e5347ef8 c01f0d10 c01f0b9c e5347f78 ee98d0c0
[ 250.825607:0] 7f00 e5347f3c e5347f10 c010734c c01f0cf0 e5347f78 00000002 de0ef180 b7a5dd34
[ 250.834035:0] 7f20 e5347f78 00000000 00000000 00000000 e5347f6c e5347f40 c00c413c c01072d0
[ 250.842538:0]
we can analysis sp(e5347ec0) or fp (e5347ecc)
#rd e5347ec0 200
crash> rd 0xe5347ec0 200
e5347ec0: e5347ef4 e5347ed0 c01f0c38 c01f05ec .~4..~4.8.......
e5347ed0: e5347f78 00000002 c01f0ce4 de0ef180 x.4.............
e5347ee0: 00000002 b7a5dd34 e5347f0c e5347ef8 ....4.....4..~4.
e5347ef0: c01f0d10 c01f0b9c e5347f78 ee98d0c0 ........x.4.....
e5347f00: e5347f3c e5347f10 c010734c c01f0cf0 <.4...4.Ls......
e5347f10: e5347f78 00000002 de0ef180 b7a5dd34 x.4.........4...
e5347f20: e5347f78 00000000 00000000 00000000 x.4.............
e5347f30: e5347f6c e5347f40 c00c413c c01072d0 [email protected].<A...r..
e5347f40: 00000000 e52ba080 de0ef180 de0ef180 ......+.........
e5347f50: b7a5dd34 00000002 00000004 00000000 4...............
e5347f60: e5347fa4 e5347f70 c00c43a0 c00c408c ..4.p.4..C...@..
e5347f70: e52ba080 00000000 00000000 00000000 ..+.............
e5347f80: c00c2468 00000003 00000002 00000001 h$..............
e5347f90: c000f5a8 e5346000 00000000 e5347fa8 .....`4.......4.
e5347fa0: c000f400 c00c4368 00000003 00000002 ....hC..........
e5347fb0: 00000001 b7a5dd34 00000002 ffffffff ....4...........
e5347fc0: 00000003 00000002 00000001 00000004 ................
e5347fd0: b7a5dd34 00000000 b7a5c7fc b6f7c418 4...............
e5347fe0: b6f77f40 bead57d8 b6f63103 b6ee39b0 @....W...1...9..
e5347ff0: 20000010 00000001 00000000 00000000 ... ............
e5348000: 00000000 00000000 00000000 00000000 ................
e5348010: 00000000 00000000 00000000 00000000 ................
e5348020: 00000000 00000000 00000000 00000000 ................
e5348030: 00000000 00000000 00000000 00000000 ................
e5348040: 00000000 00000000 00000000 00000000 ................
e5348050: 00000000 00000000 00000000 00000000 ................
e5348060: 00000000 00000000 00000000 00000000 ................
e5348070: 00000000 00000000 00000000 00000000 ................
e5348080: 00000000 00000000 00000000 00000000 ................
e5348090: 00000000 00000000 00000000 00000000 ................
e53480a0: 00000000 00000000 00000000 00000000 ................
e53480b0: 00000000 00000000 00000000 00000000 ................
e53480c0: 00000000 00000000 00000000 00000000 ................
e53480d0: 00000000 00000000 00000000 00000000 ................
e53480e0: 00000000 00000000 00000000 00000000 ................
e53480f0: 00000000 00000000 00000000 00000000 ................
e5348100: 00000000 00000000 00000000 00000000 ................
e5348110: 00000000 00000000 00000000 00000000 ................
e5348120: 00000000 00000000 00000000 00000000 ................
e5348130: 00000000 00000000 00000000 00000000 ................
e5348140: 00000000 00000000 00000000 00000000 ................
e5348150: 00000000 00000000 00000000 00000000 ................
e5348160: 00000000 00000000 00000000 00000000 ................
e5348170: 00000000 00000000 00000000 00000000 ................
e5348180: 00000000 00000000 00000000 00000000 ................
e5348190: 00000000 00000000 00000000 00000000 ................
e53481a0: 00000000 00000000 00000000 00000000 ................
e53481b0: 00000000 00000000 00000000 00000000 ................
e53481c0: 00000000 00000000 00000000 00000000 ................
e53481d0: 00000000 00000000 00000000 00000000 ................
crash> rd 0xe5347ecc 200
e5347ecc: c01f05ec e5347f78 00000002 c01f0ce4 ....x.4.........
e5347edc: de0ef180 00000002 b7a5dd34 e5347f0c ........4.....4.
e5347eec: e5347ef8 c01f0d10 c01f0b9c e5347f78 .~4.........x.4.
e5347efc: ee98d0c0 e5347f3c e5347f10 c010734c ....<.4...4.Ls..
e5347f0c: c01f0cf0 e5347f78 00000002 de0ef180 ....x.4.........
e5347f1c: b7a5dd34 e5347f78 00000000 00000000 4...x.4.........
e5347f2c: 00000000 e5347f6c e5347f40 c00c413c [email protected].<A..
e5347f3c: c01072d0 00000000 e52ba080 de0ef180 .r........+.....
e5347f4c: de0ef180 b7a5dd34 00000002 00000004 ....4...........
e5347f5c: 00000000 e5347fa4 e5347f70 c00c43a0 ......4.p.4..C..
e5347f6c: c00c408c e52ba080 00000000 00000000 .@....+.........
e5347f7c: 00000000 c00c2468 00000003 00000002 ....h$..........
e5347f8c: 00000001 c000f5a8 e5346000 00000000 .........`4.....
e5347f9c: e5347fa8 c000f400 c00c4368 00000003 ..4.....hC......
e5347fac: 00000002 00000001 b7a5dd34 00000002 ........4.......
e5347fbc: ffffffff 00000003 00000002 00000001 ................
e5347fcc: 00000004 b7a5dd34 00000000 b7a5c7fc ....4...........
e5347fdc: b6f7c418 b6f77f40 bead57d8 b6f63103 [email protected]..
e5347fec: b6ee39b0 20000010 00000001 00000000 .9..... ........
e5347ffc: 00000000 00000000 00000000 00000000 ................
e534800c: 00000000 00000000 00000000 00000000 ................
e534801c: 00000000 00000000 00000000 00000000 ................
e534802c: 00000000 00000000 00000000 00000000 ................
e534803c: 00000000 00000000 00000000 00000000 ................
e534804c: 00000000 00000000 00000000 00000000 ................
e534805c: 00000000 00000000 00000000 00000000 ................
e534806c: 00000000 00000000 00000000 00000000 ................
e534807c: 00000000 00000000 00000000 00000000 ................
e534808c: 00000000 00000000 00000000 00000000 ................
e534809c: 00000000 00000000 00000000 00000000 ................
e53480ac: 00000000 00000000 00000000 00000000 ................
e53480bc: 00000000 00000000 00000000 00000000 ................
e53480cc: 00000000 00000000 00000000 00000000 ................
e53480dc: 00000000 00000000 00000000 00000000 ................
e53480ec: 00000000 00000000 00000000 00000000 ................
e53480fc: 00000000 00000000 00000000 00000000 ................
./../../../../prebuilt/linux-x86/toolchain/arm-eabi-4.4.3/bin/arm-eabi-objdump -D vmlinux > kernel.asm
vim kernel.asm
(e5347ecc: ---> c01f05ec)
search c01d05ec from kernel.asm
and you will find handle_sysrq.
set one PC pointer, and bt to show call stack
example 2
// dissemble destroy_workqueue
crash> dis destroy_workqueue
0xc0055400 <destroy_workqueue>: mov r12, sp
0xc0055404 <destroy_workqueue+0x4>: push {r3, r4, r5, r6, r7, r8, r11, r12, lr, pc}
0xc0055408 <destroy_workqueue+0x8>: sub r11, r12, #4
0xc005540c <destroy_workqueue+0xc>: stmfd sp!, {lr}
0xc0055410 <destroy_workqueue+0x10>: ldmfd sp!, {lr}
0xc0055414 <destroy_workqueue+0x14>: mov r4, r0
0xc0055418 <destroy_workqueue+0x18>: bl 0xc005524c <drain_workqueue>
0xc005541c <destroy_workqueue+0x1c>: ldr r0, [pc, #412] ; 0xc00555c0 <destroy_workqueue+0x1c0>
0xc0055420 <destroy_workqueue+0x20>: bl 0xc05b8ee8 <_raw_spin_lock>
0xc0055424 <destroy_workqueue+0x24>: ldr r3, [r4, #12]
0xc0055428 <destroy_workqueue+0x28>: ldr r2, [r4, #8]
0xc005542c <destroy_workqueue+0x2c>: ldr r0, [pc, #396] ; 0xc00555c0 <destroy_workqueue+0x1c0>
0xc0055430 <destroy_workqueue+0x30>: ldr r6, [pc, #396] ; 0xc00555c4 <destroy_workqueue+0x1c4>
0xc0055434 <destroy_workqueue+0x34>: str r3, [r2, #4]
crash> sym 0xc0055434
c0055434 (T) destroy_workqueue+0x34
static inline void list_del(struct list_head *entry)
{
__list_del(entry->prev, entry->next);
entry->next = LIST_POISON1;
entry->prev = LIST_POISON2;
}
crash> sym udats
c0a9597c (b) udats
crash> rd c0a9597c 52
c0a9597c: 00000000 00000000 00000000 00000000 ................
c0a9598c: 00000000 00000000 00000000 00000000 ................
c0a9599c: 00000000 00000000 00000000 00000000 ................
c0a959ac: 00000000 00000000 00000000 00000000 ................
c0a959bc: 00000000 00000000 00000000 00000000 ................
c0a959cc: ef2abd00 ef2ac500 ef2acd00 00000000 ..*...*...*.....
c0a959dc: 00000000 00000000 00000000 00000000 ................
c0a959ec: 00000000 00000000 00000000 00000000 ................
c0a959fc: 00000000 ef2ad500 ef2add00 ef2ae500 ......*...*...*.
c0a95a0c: 00000000 00000000 00000000 00000000 ................
c0a95a1c: 00000000 00000000 00000000 00000000 ................
c0a95a2c: 00000000 00000000 00000000 00000000 ................
c0a95a3c: 00000000 00000000 00000000 00000000 ................
crash> struct udat ef2ad500
struct udat {
stats = {
rx_packets = 0,
tx_packets = 0,
rx_bytes = 0,
tx_bytes = 0,
rx_errors = 0,
tx_errors = 0,
rx_dropped = 0,
tx_dropped = 0,
multicast = 0,
collisions = 0,
rx_length_errors = 0,
rx_over_errors = 0,
rx_crc_errors = 0,
rx_frame_errors = 0,
rx_fifo_errors = 0,
rx_missed_errors = 0,
tx_aborted_errors = 0,
tx_carrier_errors = 0,
tx_fifo_errors = 0,
tx_heartbeat_errors = 0,
tx_window_errors = 0,
rx_compressed = 0,
tx_compressed = 0
},
netdev = 0xef2ad000,
pdata = 0xc08b8e20,
net_wq = 0xedd02880,
net_work = {
data = {
counter = 1280
},
entry = {
next = 0xef2ad56c,
prev = 0xef2ad56c
//
crash> struct work_struct 0xef2ad56c
struct work_struct {
data = {
counter = -282405524
},
entry = {
next = 0xef2ad56c,
prev = 0xc041802c
},
func = 0,
callback = 0xef2ad57c
}
crash> struct workqueue_struct 0xedd02880
struct workqueue_struct {
flags = 154,
cpu_wq = {
pcpu = 0xede55600,
single = 0xede55600,
v = 3991229952
},
list = {
next = 0x100100,
prev = 0x200200
},
flush_mutex = {
count = {
counter = 1
},
wait_lock = {
{
rlock = {
raw_lock = {
lock = 0
},
break_lock = 0,
magic = 3735899821,
owner_cpu = 4294967295,
owner = 0xffffffff
}
}
},
wait_list = {
next = 0xedd028a8,
prev = 0xedd028a8
},
owner = 0xedfbf300,
name = 0x0,
magic = 0xedd02890
},