关于单核/多核死机问题
以下信息来自https://www.cnblogs.com/jerry116/p/8799355.html。另外还未看代码进行验证
https://www.cnblogs.com/sky-heaven/p/16616530.html
linux-3.16
1、对于非抢占内核。如果代码中出现死锁（未屏蔽软中断、本cpu中断）或者死循环。那么出现死锁的这个cpu将一直卡住，无法进行任务调度。
对于这种情况需要一种检测机制去发现这种问题。具体就是每个核有一个喂狗线程（优先级最高？）和一个喂狗软中断（定时器？？）。喂狗线程负责不断更新时间戳。喂狗软中断负责检测时间戳是否更新。
即使出现了cpu无法调度的情况（不考虑中断被屏蔽）。由于软中断能够打断线程或者进程的运行。当喂狗软中断触发时，先去检查上一次喂狗线程的时间。如果当前时间距离上一次喂狗的时间超过了一定的阈值。则认为未进行任务调度，打印调用栈信息（由于任务切换不了，那么软中断打断的一定是死锁的地方）。如果时间戳未超，说明喂狗线程还能被调度，cpu还能进行任务切换
编译内核的时候需要加上这些配置
static struct smp_hotplug_thread watchdog_threads = {
.store = &softlockup_watchdog,
.thread_should_run = watchdog_should_run,
.thread_fn = watchdog,
.thread_comm = "watchdog/%u",
.setup = watchdog_enable,
.cleanup = watchdog_cleanup,
.park = watchdog_disable,
.unpark = watchdog_enable,
};
喂狗线程
lockup_detector_init会为每个cpu创建一个watchdog内核线程
void __init lockup_detector_init(void)
{
set_sample_period();
if (watchdog_user_enabled)
watchdog_enable_all_cpus(false);
}
static int watchdog_enable_all_cpus(bool sample_period_changed)
{
.................
err = smpboot_register_percpu_thread(&watchdog_threads);
........................
return err;
}__smpboot_create_thread->smpboot_thread_fn。
static int smpboot_thread_fn(void *data)
{
.......................................
set_current_state(TASK_RUNNING);
preempt_enable();
ht->thread_fn(td->cpu);
...............................
}其中ht->thread_fn就对应了回调函数watchdog。可以看到喂狗线程的回调函数更新了一下时间戳（__touch_watchdog）。
那喂狗线程何时被调度呢？下面进行解析
static void watchdog(unsigned int cpu)
{
__this_cpu_write(soft_lockup_hrtimer_cnt,
__this_cpu_read(hrtimer_interrupts));
__touch_watchdog();
}
static void __touch_watchdog(void)
{
__this_cpu_write(watchdog_touch_ts, get_timestamp());
}喂狗软中断（其实就是高精度定时器），检测cpu是否无法调度
linux3.16 watchdog_enable是在这个地方进行注册的
watchdog_enable在smpboot_thread_fn第一次执行时得到执行，随后其状态设为HP_THREAD_ACTIVE。
static void watchdog_enable(unsigned int cpu)
{
struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
/* kick off the timer for the hardlockup detector */
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer->function = watchdog_timer_fn;//高精度定时器回调函数
/* Enable the perf event */
watchdog_nmi_enable(cpu);
/* 启动一个定时器 */
/* done here because hrtimer_start can only pin to smp_processor_id() */
hrtimer_start(hrtimer, ns_to_ktime(sample_period),
HRTIMER_MODE_REL_PINNED);
/* initialize timestamp */
watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
__touch_watchdog();
}高精度定时器，看网上说都是在中断上下文中执行的（中断上半部以及软中断中）
linux 内核 高精度定时器（hrtimer）实现机制_hrtimer_start_老王不让用的博客-CSDN博客
- HRTIMER_MODE_SOFT：表示该定时器是否是“软”的，也就是定时器到期回调函数是在软中断（HRTIMER_SOFTIRQ，高精度定时器）下被执行的。
- HRTIMER_MODE_HARD：表示该定时器是否是“硬”的，也就是定时器到期回调函数是
- 在中断处理程序中被执行的。看打印好像确实是这样的 看打印好像确实是这样的
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) { ......................... printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id()); ........................ return HRTIMER_RESTART; }
从函数watchdog_enable里面可以看到高精度定时器的回调函数是watchdog_timer_fn
1、可以看到喂狗线程就是在高精度定时器里面被唤醒的。
wake_up_process(__this_cpu_read(softlockup_watchdog));
2、定时器的下一次触发时间也是在这里更新
hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
3、检测是否发生soft lockup也是在这个里面is_softlockup
/* 高精度定时器的回调是在中断上下文执行的 */
/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
struct pt_regs *regs = get_irq_regs();
int duration;
int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
/* kick the hardlockup detector */
watchdog_interrupt_count();//增加计数,这个应该是hard lockup使用
/* 唤醒喂狗线程 */
/* kick the softlockup detector */
wake_up_process(__this_cpu_read(softlockup_watchdog));
/* 增加高精度定时器的超时时间,并重启定时器 */
/* .. and repeat */
hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
if (touch_ts == 0) {//为什么时间戳会是0呢？难道是第一次执行？？
if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
/*
* If the time stamp was touched atomically
* make sure the scheduler tick is up to date.
*/
__this_cpu_write(softlockup_touch_sync, false);
sched_clock_tick();
}
/* Clear the guest paused flag on watchdog reset */
kvm_check_and_clear_guest_paused();
__touch_watchdog();
return HRTIMER_RESTART;
}
/* 现在时间到上次时间戳的位置已经超了 */
duration = is_softlockup(touch_ts);
if (unlikely(duration)) {//非0说明已经触发检测机制,cpu在规定时间内未进行调度
if (kvm_check_and_clear_guest_paused())
return HRTIMER_RESTART;
/* only warn once */
if (__this_cpu_read(soft_watchdog_warn) == true)
return HRTIMER_RESTART;
if (softlockup_all_cpu_backtrace) {
if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
/* Someone else will report us. Let's give up */
__this_cpu_write(soft_watchdog_warn, true);
return HRTIMER_RESTART;
}
}
printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
smp_processor_id(), duration,
current->comm, task_pid_nr(current));
print_modules();
print_irqtrace_events(current);
if (regs)
show_regs(regs);
else
dump_stack();
if (softlockup_all_cpu_backtrace) {
trigger_allbutself_cpu_backtrace();
clear_bit(0, &soft_lockup_nmi_warn);
/* Barrier to sync with other cpus */
smp_mb__after_atomic();
}
if (softlockup_panic)
panic("softlockup: hung tasks");
__this_cpu_write(soft_watchdog_warn, true);
} else
__this_cpu_write(soft_watchdog_warn, false);
return HRTIMER_RESTART;
}
static int is_softlockup(unsigned long touch_ts)
{
unsigned long now = get_timestamp();
/* 当前时间已经超过上一次的时间戳 + 检测阈值 */
/* Warn about unreasonable delays: */
if (time_after(now, touch_ts + get_softlockup_thresh()))
return now - touch_ts;
return 0;
}可以看到基于软件实现的喂狗原理大致如下：
首先存在一个喂狗线程，和一个喂狗软中断（其实就是高精度定时器）。其中喂狗线程被调度的时候就更新时间戳watchdog_touch_ts（per cpu变量，每个cpu都有一个）。喂狗定时器里面负责唤醒喂狗线程去更新时间戳，并且检查当前时间和时间戳的差值。如果差值超过一定的阈值，则认为触发soft lockup.
代码示例：非抢占内核
从调用栈也能看到是test_thread里面出了问题
创建了一个内核线程进行死循环
static struct task_struct *test_task; int test_thread(void* a) { unsigned long flags; printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id()); while (1){} return 0; }softlockup simulate, in_interrupt 0 in_softirq 0, cpu id 3 BUG: soft lockup - CPU#3 stuck for 23s! [test_task:586] Modules linked in: CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41 task: ee01c800 ti: ee206000 task.ti: ee206000 PC is at test_thread+0x30/0x38 LR is at test_thread+0x30/0x38 pc : [] lr : [ ] psr: 60000013 sp : ee207f60 ip : 00000001 fp : 00000000 r10: 00000000 r9 : 00000000 r8 : 00000000 r7 : c0305d28 r6 : 00000000 r5 : 00000000 r4 : ee1fd400 r3 : 000004f0 r2 : c089f494 r1 : 20000093 r0 : 0000003d Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment kernel Control: 10c53c7d Table: 8e2f806a DAC: 00000015 CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41 [ ] (unwind_backtrace) from [ ] (show_stack+0x10/0x14) [ ] (show_stack) from [ ] (dump_stack+0x74/0x90) [ ] (dump_stack) from [ ] (watchdog_timer_fn+0x158/0x1bc) [ ] (watchdog_timer_fn) from [ ] (hrtimer_run_queues+0xcc/0x23c) [ ] (hrtimer_run_queues) from [ ] (run_local_timers+0x8/0x14) [ ] (run_local_timers) from [ ] (update_process_times+0x2c/0x58) [ ] (update_process_times) from [ ] (tick_periodic+0x34/0xbc) [ ] (tick_periodic) from [ ] (tick_handle_periodic+0x2c/0x94) [ ] (tick_handle_periodic) from [ ] (twd_handler+0x2c/0x40) [ ] (twd_handler) from [ ] (handle_percpu_devid_irq+0x68/0x84) [ ] (handle_percpu_devid_irq) from [ ] (generic_handle_irq+0x20/0x30) [ ] (generic_handle_irq) from [ ] (handle_IRQ+0x38/0x94) [ ] (handle_IRQ) from [ ] (gic_handle_irq+0x28/0x5c) [ ] (gic_handle_irq) from [ ] (__irq_svc+0x40/0x50) Exception stack(0xee207f18 to 0xee207f60) 7f00: 0000003d 20000093 7f20: c089f494 000004f0 ee1fd400 00000000 00000000 c0305d28 00000000 00000000 7f40: 00000000 00000000 00000001 ee207f60 c0305d58 c0305d58 60000013 ffffffff [ ] (__irq_svc) from [ ] (test_thread+0x30/0x38) [ ] (test_thread) from [ ] (kthread+0xcc/0xe8) [ ] (kthread) from [ ] (ret_from_fork+0x14/0x3c) Kernel panic - not syncing: softlockup: hung tasks CPU: 3 PID: 586 Comm: test_task Not tainted 3.16.0 #41 [ ] (unwind_backtrace) from [ ] (show_stack+0x10/0x14) [ ] (show_stack) from [ ] (dump_stack+0x74/0x90) [ ] (dump_stack) from [ ] (panic+0x90/0x204) [ ] (panic) from [ ] (watchdog_timer_fn+0x1a0/0x1bc) [ ] (watchdog_timer_fn) from [ ] (hrtimer_run_queues+0xcc/0x23c) [ ] (hrtimer_run_queues) from [ ] (run_local_timers+0x8/0x14) [ ] (run_local_timers) from [ ] (update_process_times+0x2c/0x58) [ ] (update_process_times) from [ ] (tick_periodic+0x34/0xbc) [ ] (tick_periodic) from [ ] (tick_handle_periodic+0x2c/0x94) [ ] (tick_handle_periodic) from [ ] (twd_handler+0x2c/0x40) [ ] (twd_handler) from [ ] (handle_percpu_devid_irq+0x68/0x84) [ ] (handle_percpu_devid_irq) from [ ] (generic_handle_irq+0x20/0x30) [ ] (generic_handle_irq) from [ ] (handle_IRQ+0x38/0x94) [ ] (handle_IRQ) from [ ] (gic_handle_irq+0x28/0x5c) [ ] (gic_handle_irq) from [ ] (__irq_svc+0x40/0x50) Exception stack(0xee207f18 to 0xee207f60) 7f00: 0000003d 20000093 7f20: c089f494 000004f0 ee1fd400 00000000 00000000 c0305d28 00000000 00000000 7f40: 00000000 00000000 00000001 ee207f60 c0305d58 c0305d58 60000013 ffffffff [ ] (__irq_svc) from [ ] (test_thread+0x30/0x38) [ ] (test_thread) from [ ] (kthread+0xcc/0xe8) [ ] (kthread) from [ ] (ret_from_fork+0x14/0x3c) CPU0: stopping CPU: 0 PID: 0 Comm: swapper/0 Not tainted 3.16.0 #41 [ ] (unwind_backtrace) from [ ] (show_stack+0x10/0x14) [ ] (show_stack) from [ ] (dump_stack+0x74/0x90) [ ] (dump_stack) from [ ] (handle_IPI+0x134/0x170) [ ] (handle_IPI) from [ ] (gic_handle_irq+0x54/0x5c) [ ] (gic_handle_irq) from [ ] (__irq_svc+0x40/0x50) Exception stack(0xc088ff60 to 0xc088ffa8) ff60: ffffffed 00000000 ffffffed 00000000 c088e000 00000000 00000000 c0896464 ff80: c0463dc4 00000000 c088cb30 0000004c 00000000 c088ffa8 c000efd4 c000efd8 ffa0: 60000013 ffffffff [ ] (__irq_svc) from [ ] (arch_cpu_idle+0x28/0x30) [ ] (arch_cpu_idle) from [ ] (cpu_startup_entry+0x1ac/0x1f0) [ ] (cpu_startup_entry) from [ ] (start_kernel+0x328/0x38c) CPU1: stopping CPU: 1 PID: 0 Comm: swapper/1 Not tainted 3.16.0 #41 [ ] (unwind_backtrace) from [ ] (show_stack+0x10/0x14) [ ] (show_stack) from [ ] (dump_stack+0x74/0x90) [ ] (dump_stack) from [ ] (handle_IPI+0x134/0x170) [ ] (handle_IPI) from [ ] (gic_handle_irq+0x54/0x5c) [ ] (gic_handle_irq) from [ ] (__irq_svc+0x40/0x50) Exception stack(0xee891f90 to 0xee891fd8) 1f80: ffffffed 00000000 ffffffed 00000000 1fa0: ee890000 00000000 00000000 c0896464 c0463dc4 00000000 c088cb30 0000004c 1fc0: 00000000 ee891fd8 c000efd4 c000efd8 60000013 ffffffff [ ] (__irq_svc) from [ ] (arch_cpu_idle+0x28/0x30) [ ] (arch_cpu_idle) from [ ] (cpu_startup_entry+0x1ac/0x1f0) [ ] (cpu_startup_entry) from [<60008684>] (0x60008684) CPU2: stopping CPU: 2 PID: 0 Comm: swapper/2 Not tainted 3.16.0 #41 [ ] (unwind_backtrace) from [ ] (show_stack+0x10/0x14) [ ] (show_stack) from [ ] (dump_stack+0x74/0x90) [ ] (dump_stack) from [ ] (handle_IPI+0x134/0x170) [ ] (handle_IPI) from [ ] (gic_handle_irq+0x54/0x5c) [ ] (gic_handle_irq) from [ ] (__irq_svc+0x40/0x50) Exception stack(0xee893f90 to 0xee893fd8) 3f80: ffffffed 00000000 ffffffed 00000000 3fa0: ee892000 00000000 00000000 c0896464 c0463dc4 00000000 c088cb30 0000004c 3fc0: 00000000 ee893fd8 c000efd4 c000efd8 60000013 ffffffff [ ] (__irq_svc) from [ ] (arch_cpu_idle+0x28/0x30) [ ] (arch_cpu_idle) from [ ] (cpu_startup_entry+0x1ac/0x1f0) [ ] (cpu_startup_entry) from [<60008684>] (0x60008684) PANIC: softlockup: hung tasks Entering kdb (current=0xee01c800, pid 586) on processor 3 due to Keyboard Entry [3]kdb>
2、单核挂死并屏蔽了本cpu中断的情况。eg local_irq_save, spin_lock_irqsave。对于这种情况由于CPU无法接受到中断信息了。显然中断都无法接收了，上面第一种方法就失效了。但是某些芯片有不可屏蔽中断NMI。
在多核系统里面，每个核都可以去检测其他核的中断接受情况，如果检测到某个核未接受中断了，就可以给该核发生一个不可屏蔽中断的消息，同样在这个中断处理函数里面把调用栈打出来。
该方法有部分也是和方法1公用的
watchdog_enable->watchdog_nmi_enable
static void watchdog_enable(unsigned int cpu)
{
struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
....................
/* Enable the perf event */
watchdog_nmi_enable(cpu);
..............................
}watchdog_nmi_enable里面注册了一个hard lockup检测事件。其回调函数是watchdog_overflow_callback
我感觉可能arm也是这个样子把。看x86就是每个一段时间发出一个NMI中断。然后在回调函数里面检查中断触发的次数是否增加吧
这个硬件在x86里叫performance monitoring，这个硬件有一个功能就是在cpu clock经过了多少个周期后发出一个NMI中断出来。
static int watchdog_nmi_enable(unsigned int cpu)
{
......................
wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
/* Try to register using hardware perf events */
event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);
............................
}hard lockup判定：watchdog_overflow_callback->is_hardlockup
is_hardlockup：可以看到它先去读取中断被触发的次数。然后再去比较上一次NMI中断触发时保存的中断次数（hrtimer_interrupts_saved）。如果相等说明出现了hard lockup.
另外hrtimer_interrupts这个变量在方法1的喂狗软中断里面就会更新。
static int is_hardlockup(void)
{
unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
return 1;
__this_cpu_write(hrtimer_interrupts_saved, hrint);
return 0;
}下面这种hard lockup怎么构造不出来呢？ 反而一直报rcu相关的问题
int test_thread(void* a) { unsigned long flags; printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id()); local_irq_disable(); while (1){} return 0; }
[root@arm_test ]# INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=8407 jiffies, g=-62, c=-63, q=308)
Task dump for CPU 1:
test_task R running 0 653 2 0x00000002
[] (__schedule) from [ ] (kthread+0xcc/0xe8)
[] (kthread) from [ ] (ret_from_fork+0x14/0x3c)
INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=14712 jiffies, g=-62, c=-63, q=308)
Task dump for CPU 1:
test_task R running 0 653 2 0x00000002
[] (__schedule) from [ ] (kthread+0xcc/0xe8)
[] (kthread) from [ ] (ret_from_fork+0x14/0x3c)
通过top和interrupt能看到是哪个进程出问题了。
softlockup simulate, in_interrupt 0 in_softirq 0, cpu id 1//内核线程在1核上
[root@arm_test ]# cat /proc/interrupts
CPU0 CPU1 CPU2 CPU3
29: 9131 1444 9128 9112 GIC 29 twd
root@arm_test ]# cat /proc/interrupts
CPU0 CPU1 CPU2 CPU3
29: 9357 1444 9350 9332 GIC 29 twd
34: 6 0 0 0 GIC 34 timer
可以看到cpu1的twd中断不增加top可以看到test_task一直占用cpu1
Mem: 19528K used, 1013232K free, 0K shrd, 252K buff, 8944K cached
CPU: 0.3% usr 1.5% sys 0.0% nic 96.4% idle 0.0% io 0.0% irq 1.5% sirq
Load average: 1.65 0.55 0.19 2/54 787
PID PPID USER STAT VSZ %VSZ CPU %CPU COMMAND
653 2 0 RW 0 0.0 1 3.2 [test_task]
787 781 0 R 2432 0.2 0 0.2 top
7 2 0 SW 0 0.0 2 0.0 [rcu_sched]
后面知道为啥构造不出来了
hard lockup：需要CPU支持NMI（不可屏蔽中断，通常是通过CPU里的PMU单元实现的），如果PMU发现长时间（这个cycle是借助NMI来计算的，因为定时器可能不工作了）一个中断都不来，就知道发生了hard lockup，这时（触发NMI中断，中断处理函数中）分析栈就知道在哪里锁住中断的。需要把CONFIG_HARDLOCKUP_DETECTOR打开。
由于ARM里面没有NMI，因此内核不支持ARM的hard lockup detector。但有一些内核patch可以用，比如用FIQ模拟MNI（如果FIQ用于其他地方了，这里就用不了了），或者用CPU1去检测CPU0是否被hard lockup(但CPU1没办法获得线程的栈，只能知道lockup了)，但这两个patch都没在主线上。FIQ在Linux中基本不用的（一般只做特殊的debugger，常规代码不用）。
————————————————
版权声明：本文为CSDN博主「落尘纷扰」的原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/jasonchen_gbd/article/details/79465405
如果没有这种不可屏蔽中断的芯片（例如arm），这种怎么处理呢？？要不明天试试自己构造一个检测的
还是利用is_hardlockup里面的两个每cpu变量。起一个高精度定时器。每隔一段时间检查一下这两个值是不是相等。如果是相等的说明该cpu关中断了。唯一需要注意的是：假设cpu0关中断了，它自己是不能发现的，只有其他cpu帮忙检查才行。
主要代码如下watch_dog.c：
主要使用者三个变量
/*****************************************/
static DEFINE_PER_CPU(struct hrtimer, hardlock_check_hrtimer);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
/*****************************************/watchdog_enable里面再起一个定时器，检查是否有hardlockup
static void watchdog_enable(unsigned int cpu)
{
.............................................................
/* 添加hardlock定时器,用于检查hardlock的情况 */
hrtimer = &__raw_get_cpu_var(hardlock_check_hrtimer);
/* kick off the timer for the hardlockup detector */
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer->function = hardlock_check_callback;
__this_cpu_write(hrtimer_interrupts_saved, 0);
__this_cpu_write(hrtimer_interrupts, 0);
/* done here because hrtimer_start can only pin to smp_processor_id() */
hrtimer_start(hrtimer, ns_to_ktime(sample_period * 2),
HRTIMER_MODE_REL_PINNED);
}定时器回调函数
static void hardlock_check_callback(struct hrtimer *hrtimer)
{
/* check for a hardlockup
* This is done by making sure our timer interrupt
* is incrementing. The timer interrupt should have
* fired multiple times before we overflow'd. If it hasn't
* then this is a good indication the cpu is stuck
*/
if (is_hardlockup()) {
int this_cpu = smp_processor_id();
WARN(1, "CPU %d happen hard LOCKUP detect by cpu %d", (this_cpu + 1) % 4, this_cpu);
//return;
}
/* 增加高精度定时器的超时时间,并重启定时器 */
/* .. and repeat */
hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period * 2));
return;
}watchdog_timer_fn->watchdog_interrupt_count:这个里面会更新这个值的，所以我们不用管自己用即可hrtimer_interrupts
cpu0->cpu1->cpu2->cpu3。cpu3反过来监控cpu0的计数
static int is_hardlockup(void)
{
/* 每次进来先更新当前cpu的中断计数 */
unsigned long cur_hrint = __this_cpu_read(hrtimer_interrupts);
__this_cpu_write(hrtimer_interrupts_saved, cur_hrint);
/* 读取需要监控的cpu的计数，判断是否出现hardlock */
int cpu = smp_processor_id();
int check_cpu = (cpu + 1) % 4;
unsigned long hrint = per_cpu(hrtimer_interrupts, check_cpu);
unsigned long saved_hrint = per_cpu(hrtimer_interrupts_saved, check_cpu);
if (saved_hrint == hrint)
{
return 1;
}
/*
更新监控的cpu计数
当我监控完之后，该cpu出现了一直关中断的情况，如果不更新，那么interrupts和saved是一直不相等的
*/
per_cpu(hrtimer_interrupts_saved, check_cpu) = hrint;
return 0;
}测试代码
int test_thread(void* a)
{
unsigned long flags;
printk(KERN_EMERG "\r\n softlockup simulate, in_interrupt %u in_softirq %u, cpu id %d\n", in_interrupt(), in_softirq(), smp_processor_id());
local_irq_disable();
while (1){}
//f1(10, 20);
return 0;
}运行在cpu1上，cpu0负责监控cpu1（实测有误报的情况）
3、对于所有核都挂死的情况。可以借用狗叫重启设备。异常现场信息记录，可以在挂死前把所有的寄存器记录下来。