watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]
测试代码 如下：
#include
#include
#include
static DEFINE_SPINLOCK(hack_spinA);
static DEFINE_SPINLOCK(hack_spinB);
void hack_spinAB(void)
{
printk("hack_lockdep:A->B\n");
spin_lock(&hack_spinA);
spin_lock(&hack_spinB);
}
void hack_spinBA(void)
{
printk("hack_lockdep:B->A\n");
spin_lock(&hack_spinB);
}
static int __init lockdep_test_init(void)
{
printk("figo:my lockdep module init\n");
hack_spinAB();
hack_spinBA();
return 0;
}
static void __exit lockdep_test_exit(void)
{
printk("goodbye\n");
}
module_init(lockdep_test_init);
module_exit(lockdep_test_exit);
MODULE_LICENSE("GPL");
[ 28.015539] -(3)[1:swapper/0->0:swapper/0]watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]
[ 28.025300] -(3)[1:swapper/0->0:swapper/0]Kernel panic - not syncing: softlockup: hung tasks
[ 28.033776] -(3)[1:swapper/0->0:swapper/0]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2
[ 28.043906] -(3)[1:swapper/0->0:swapper/0]Hardware name: Platfrom(DT)
[ 28.050636] -(3)[1:swapper/0->0:swapper/0]Call trace:
[ 28.055711] -(3)[1:swapper/0->0:swapper/0] dump_backtrace+0x0/0x198
[ 28.062006] -(3)[1:swapper/0->0:swapper/0] show_stack+0x20/0x2c
[ 28.067951] -(3)[1:swapper/0->0:swapper/0] dump_stack+0xd8/0x128
[ 28.073984] -(3)[1:swapper/0->0:swapper/0] panic+0x134/0x334
[ 28.079669] -(3)[1:swapper/0->0:swapper/0] softlockup_fn+0x0/0x60
[ 28.085790] -(3)[1:swapper/0->0:swapper/0] __run_hrtimer+0xa8/0x2b0
[ 28.092085] -(3)[1:swapper/0->0:swapper/0] hrtimer_interrupt+0x174/0x3c8
[ 28.098817] -(3)[1:swapper/0->0:swapper/0] arch_timer_handler_phys+0x40/0x50
[ 28.105897] -(3)[1:swapper/0->0:swapper/0] handle_percpu_devid_irq+0x88/0x278
[ 28.113065] -(3)[1:swapper/0->0:swapper/0] __handle_domain_irq+0x84/0xc4
[ 28.119796] -(3)[1:swapper/0->0:swapper/0] gic_handle_irq+0x10c/0x184
[ 28.126265] -(3)[1:swapper/0->0:swapper/0] el1_irq+0xec/0x198
[ 28.132035] -(3)[1:swapper/0->0:swapper/0] queued_spin_lock_slowpath+0x94/0x270
[ 28.139376] -(3)[1:swapper/0->0:swapper/0] _raw_spin_lock+0x50/0x54
[ 28.145670] -(3)[1:swapper/0->0:swapper/0] lockdep_test_init+0x34/0x40
[ 28.152226] -(3)[1:swapper/0->0:swapper/0] do_one_initcall+0xa0/0x298
[ 28.158694] -(3)[1:swapper/0->0:swapper/0] do_initcall_level+0xc0/0x164
[ 28.165337] -(3)[1:swapper/0->0:swapper/0] do_basic_setup+0x34/0x4c
[ 28.171631] -(3)[1:swapper/0->0:swapper/0] kernel_init_freeable+0xd0/0x150
[ 28.178535] -(3)[1:swapper/0->0:swapper/0] kernel_init+0x18/0x29c
[ 28.184655] -(3)[1:swapper/0->0:swapper/0] ret_from_fork+0x10/0x18
[ 28.190864] -(3)[1:swapper/0->0:swapper/0]SMP: stopping secondary CPUs
[ 28.197425] -(3)[1:swapper/0->0:swapper/0]Kernel Offset: disabled
[ 28.203545] -(3)[1:swapper/0->0:swapper/0]CPU features: 0x00000000,22002a38
[ 28.210537] -(3)[1:swapper/0->0:swapper/0]Memory Limit: none
[ 28.216221] -(3)[1:swapper/0->0:swapper/0]eskdump die
[ 28.221295] -(3)[1:swapper/0->0:swapper/0]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2
[ 28.231425] -(3)[1:swapper/0->0:swapper/0]Hardware name: ES (DT)
[ 28.238156] -(3)[1:swapper/0->0:swapper/0]pstate: 60000089 (nZCv daIf -PAN -UAO)
[ 28.245584] -(3)[1:swapper/0->0:swapper/0]pc : panic+0x1b8/0x334
[ 28.251617] -(3)[1:swapper/0->0:swapper/0]lr : panic+0x148/0x334
[ 28.257649] -(3)[1:swapper/0->0:swapper/0]sp : ffff00000801bbe0
[ 28.263592] -(3)[1:swapper/0->0:swapper/0]x29: ffff00000801bd80 x28: 0000000000000001
[ 28.271543] -(3)[1:swapper/0->0:swapper/0]x27: ffff80037ff9d180 x26: 0000000685682d4d
[ 28.279494] -(3)[1:swapper/0->0:swapper/0]x25: ffff00000a4fd000 x24: 0000000000000003
[ 28.287445] -(3)[1:swapper/0->0:swapper/0]x23: ffff00000a0208b0 x22: 0000000000000000
[ 28.295396] -(3)[1:swapper/0->0:swapper/0]x21: ffff800379219ec0 x20: 0000000000000000
[ 28.303346] -(3)[1:swapper/0->0:swapper/0]x19: ffff000009af821d x18: 0000000000000001
[ 28.311297] -(3)[1:swapper/0->0:swapper/0]x17: 00000000000cc70c x16: 00000000000000f4
[ 28.319248] -(3)[1:swapper/0->0:swapper/0]x15: ffff00000923f160 x14: ffff00000980e250
[ 28.327199] -(3)[1:swapper/0->0:swapper/0]x13: 000000000000369a x12: 0000000029aaaaab
[ 28.335150] -(3)[1:swapper/0->0:swapper/0]x11: 0000000000000001 x10: 0000000006666998
[ 28.343100] -(3)[1:swapper/0->0:swapper/0]x9 : 4b602900c22d1900 x8 : ffff00000801bc20
[ 28.351051] -(3)[1:swapper/0->0:swapper/0]x7 : 73203a504d535d30 x6 : ffff00000a8b5b16
[ 28.359002] -(3)[1:swapper/0->0:swapper/0]x5 : 0000000000000001 x4 : 000000000000000c
[ 28.366953] -(3)[1:swapper/0->0:swapper/0]x3 : 000000000000004a x2 : 0000000000000003
[ 28.374903] -(3)[1:swapper/0->0:swapper/0]x1 : 0000000000000004 x0 : 0000000000000001
[ 28.382854] -(3)[1:swapper/0->0:swapper/0]Modules linked in:
[ 28.397593] -(3)[1:swapper/0->0:swapper/0]done available data size 0x83000
[ 28.404503] -(3)[1:swapper/0->0:swapper/0]kdump_sign sig 0x8a6acdb4, size 0x83000
[ 28.412017] -(3)[1:swapper/0->0:swapper/0]rebooting...
这里没有在kernel 添加lockdep配置，打印的calltrace信息，测试代码逻辑比较简单，所以能一眼看出来lockdep_test_init中有问题
Linux内核中打开lockdep功能，需要打开CONFIG_DEBUG_LOCKDEP选项：
CONFIG_LOCK_STAT=y
CONFIG_PROVE_LOCKING=y
CONFIG_DEBUG_LOCKDEP=yconsole:/ # insmod ./lockdep_test.ko
[ 501.016783] -(3)[2844:insmod->2519:sh]
[ 501.020560] -(3)[2844:insmod->2519:sh]============================================
[ 501.028164] -(3)[2844:insmod->2519:sh]WARNING: possible recursive locking detected
[ 501.035768] -(3)[2844:insmod->2519:sh]4.19.176 #1 Tainted: G O
[ 501.042935] -(3)[2844:insmod->2519:sh]--------------------------------------------
[ 501.050538] -(3)[2844:insmod->2519:sh]insmod/2844 is trying to acquire lock:
[ 501.057617] 00000000f287eb44 (hack_spinB){+.+.}, at: init_module+0x34/0xfb0 [lockdep_test]
[ 501.065931] -(3)[2844:insmod->2519:sh]
[ 501.065931] but task is already holding lock:
[ 501.074056] 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[ 501.082275] -(3)[2844:insmod->2519:sh]
[ 501.082275] other info that might help us debug this:
[ 501.091100] -(3)[2844:insmod->2519:sh] Possible unsafe locking scenario:
[ 501.091100]
[ 501.099313] -(3)[2844:insmod->2519:sh] CPU0
[ 501.104038] -(3)[2844:insmod->2519:sh] ----
[ 501.108761] -(3)[2844:insmod->2519:sh] lock(hack_spinB);
[ 501.114184] -(3)[2844:insmod->2519:sh] lock(hack_spinB);
[ 501.119608] -(3)[2844:insmod->2519:sh]
[ 501.119608] *** DEADLOCK ***
[ 501.119608]
[ 501.127823] -(3)[2844:insmod->2519:sh] May be due to missing lock nesting notation
[ 501.127823]
[ 501.136908] -(3)[2844:insmod->2519:sh]2 locks held by insmod/2844:
[ 501.143114] -(3)[2844:insmod->2519:sh] #0: 000000000721ea50 (hack_spinA){+.+.}, at: hack_spinAB+0x24/0x38 [lockdep_test]
[ 501.154035] -(3)[2844:insmod->2519:sh] #1: 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[ 501.164958] -(3)[2844:insmod->2519:sh]
[ 501.164958] stack backtrace:
[ 501.171605] -(3)[2844:insmod->2519:sh]CPU: 3 PID: 2844 Comm: insmod Tainted: G O 4.19.176 #1
[ 501.181388] -(3)[2844:insmod->2519:sh]Hardware name: ESWIN_LD60 (DT)
[ 501.187770] -(3)[2844:insmod->2519:sh]Call trace:
[ 501.192500] -(3)[2844:insmod->2519:sh] dump_backtrace+0x0/0x198
[ 501.198445] -(3)[2844:insmod->2519:sh] show_stack+0x20/0x2c
[ 501.204045] -(3)[2844:insmod->2519:sh] dump_stack+0xd8/0x128
[ 501.209732] -(3)[2844:insmod->2519:sh] __lock_acquire+0x17d8/0x1ed0
[ 501.216028] -(3)[2844:insmod->2519:sh] lock_acquire+0x12c/0x1f4
[ 501.221976] -(3)[2844:insmod->2519:sh] _raw_spin_lock+0x54/0x90
[ 501.227922] -(3)[2844:insmod->2519:sh] init_module+0x34/0xfb0 [lockdep_test]
[ 501.235003] -(3)[2844:insmod->2519:sh] do_one_initcall+0x13c/0x3e0
[ 501.241213] -(3)[2844:insmod->2519:sh] do_init_module+0x5c/0x218
[ 501.247248] -(3)[2844:insmod->2519:sh] load_module+0x296c/0x31d4
[ 501.253283] -(3)[2844:insmod->2519:sh] __se_sys_finit_module+0xd8/0x114
[ 501.259928] -(3)[2844:insmod->2519:sh] __arm64_sys_finit_module+0x20/0x2c
[ 501.266748] -(3)[2844:insmod->2519:sh] el0_svc_common+0xa0/0x170
[ 501.272782] -(3)[2844:insmod->2519:sh] el0_svc_compat_handler+0x28/0x34
[ 501.279425] -(3)[2844:insmod->2519:sh] el0_svc_compat+0x8/0x34