从一个softlock问题来谈谈Kernel IPI的实现

原创

cdh

修改于 2020-07-29 22:23:26

5.4K30

代码可运行

文章被收录于专栏：笔记+笔记+

运行总次数：0

代码可运行

问题环境：

X86-64 架构，Kernel Ver：Centos7 3.10.0-693.el7.x86_64

从bt栈信息可以看到CPU2 卡在 smp_call_function_many函数中导致watchdog超时触发系统重启：

crash> bt

PID: 12263 TASK: ffff8803bbebdee0 CPU: 2 COMMAND: "kworker/u8:0"

bt: page excluded: kernel virtual address: ffffffffffffffff type: "cpu_online_map"

#0 [ffff88043fd03cf8] machine_kexec at ffffffff8105c4cb

#1 [ffff88043fd03d58] __crash_kexec at ffffffff81104a32

#2 [ffff88043fd03e28] panic at ffffffff8169dc5f

#3 [ffff88043fd03ea8] watchdog_timer_fn at ffffffff8112f651

#4 [ffff88043fd03ee0] __hrtimer_run_queues at ffffffff810b4ae4

#5 [ffff88043fd03f38] hrtimer_interrupt at ffffffff810b507f

#6 [ffff88043fd03f80] local_apic_timer_interrupt at ffffffff81053895

#7 [ffff88043fd03f98] smp_apic_timer_interrupt at ffffffff816b76bd

#8 [ffff88043fd03fb0] apic_timer_interrupt at ffffffff816b5c1d

--- <IRQ stack> ---

#9 [ffff88000399b738] apic_timer_interrupt at ffffffff816b5c1d

[exception RIP: smp_call_function_many+514]

RIP: ffffffff810f99a2 RSP: ffff88000399b7e8 RFLAGS: 00000202

RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff88043fd9adb8

RDX: 0000000000000003 RSI: 0000000000000004 RDI: 0000000000000000

RBP: ffff88000399b820 R8: ffff88017a1ee000 R9: ffffffff813227d9

R10: ffff88043fd19c80 R11: ffffea00000c2100 R12: 0000000000000292

R13: ffff88000399b798 R14: ffffea0010b1f842 R15: 0000000000000000

ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018

#10 [ffff88000399b828] native_flush_tlb_others at ffffffff8106e668

#11 [ffff88000399b878] flush_tlb_page at ffffffff8106e864

结合smp_call_function_many函数反汇编以及源码推测CPU2是在csd_lock_wait中一直执行while语句：

crash> dis smp_call_function_many

...

0xffffffff810f99a0 <smp_call_function_many+512>: pause

0xffffffff810f99a2 <smp_call_function_many+514>: testb $0x1,0x20(%rcx)

0xffffffff810f99a6 <smp_call_function_many+518>: jne 0xffffffff810f99a0 <smp_call_function_many+512>

...

void smp_call_function_many(const struct cpumask *mask,
                            smp_call_func_t func, void *info, bool wait)
{
 ...
if (wait) {
//按顺序等各个接收到本cpu ipi信号的其他cpu修改csd的flag
                for_each_cpu(cpu, cfd->cpumask) {
                        struct call_single_data *csd;

                        csd = per_cpu_ptr(cfd->csd, cpu);
                //等待percpu变量csd的flag被置为~CSD_FLAG_LOCK,代表对应cpu已经响应并处理完本
                 cpu发送的ipi回调
                        csd_lock_wait(csd);   
                }
        }
....
}
enum {
        CSD_FLAG_LOCK           = 0x01,
        CSD_FLAG_WAIT           = 0x02,
};
static void csd_lock_wait(struct call_single_data *csd)
{
        while (csd->flags & CSD_FLAG_LOCK)
                cpu_relax();
}

接收到IPI信号的CPU执行完回调函数后会对csd的flags清零：

static void flush_smp_call_function_queue(bool warn_cpu_offline)
{
....
llist_for_each_entry_safe(csd, csd_next, entry, llist) {
                csd->func(csd->info);//执行ipi回调函数
                csd_unlock(csd);//对csd的flags清零
        }
...
}
static void csd_unlock(struct call_single_data *csd)
{
  ...      
  csd->flags &= ~CSD_FLAG_LOCK;
}

要证明我们上面的推测是成立的，那么需要找出call_singel_data类型指针变量csd的地址：

void smp_call_function_many(const struct cpumask *mask,
                            smp_call_func_t func, void *info, bool wait)
{
  ...
 cfd = &__get_cpu_var(cfd_data);//获取当前cpu的cfd_data地址
  ...
 if (wait) {
                for_each_cpu(cpu, cfd->cpumask) {
                        struct call_single_data *csd;

                        csd = per_cpu_ptr(cfd->csd, cpu);//从cfd中获取per-cpu变量csd
                        csd_lock_wait(csd);
                }
        }
}
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))

问题设备一共有四个cpu：


crash> p nr_cpu_ids
nr_cpu_ids = $2 = 4
crash> 
crash> px cfd_data
PER-CPU DATA TYPE:
  struct call_function_data cfd_data;
PER-CPU ADDRESSES:
  [0]: ffff88043fc17840
  [1]: ffff88043fc97840
  [2]: ffff88043fd17840
  [3]: ffff88043fd97840
crash>

阻塞在smp_call_function_many函数中的是cpu2：

crash> px cfd_data:2
per_cpu(cfd_data, 2) = $3 = {
  csd = 0x1adb8, 
  cpumask = 0xffff88017a1ee000
}
crash>

per_cpu(cfd_data,2)中cpumask的值是10,10的值转为二进制就是bit3和bit1值为1，对应cpu3和cpu1。

crash> struct cpumask 0xffff88017a1ee000

struct cpumask {

bits = {10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

}

crash>

crash> eval -b 10

hexadecimal: a

decimal: 10

octal: 12

binary: 0000000000000000000000000000000000000000000000000000000000001010

bits set: 3 1

crash>

per_cpu(cfd_data,2)中cpumask对应的cpu位其实就代表本cpu给哪些cpu发送了ipi信号，在

本问题的vmcore中代表cpu2给cpu1和cpu3发送了IPI信号：

void smp_call_function_many(const struct cpumask *mask,
                            smp_call_func_t func, void *info, bool wait)
{
      struct call_function_data *cfd;
        ...
        cfd = &__get_cpu_var(cfd_data);//获取本cpu对应的cfd_data地址
        cpumask_and(cfd->cpumask, mask, cpu_online_mask);//cfd->cpumask=mask&cpu_online_mask
        cpumask_clear_cpu(this_cpu, cfd->cpumask);//清除本cpu对应的掩码位，也就是不用给自己发送ipi
        ...
       for_each_cpu(cpu, cfd->cpumask) {//对percpu变量csd进行初始化
                struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
                csd_lock(csd);
                csd->func = func;//收到ipi信号的cpu执行的回调函数
                csd->info = info;
                //发起ipi的一方，通过构造一个csd挂到对方cpu的call_single_queue percpu变量中
                //csd->llist添加到call_single_queue对应中
                llist_add(&csd->llist, &per_cpu(call_single_queue, cpu));
        }

        /* Send a message to all CPUs in the map */
        arch_send_call_function_ipi_mask(cfd->cpumask);//给cpumask中对应bit位的cpu发送ipi信号

        if (wait) {
             //按顺序等各个接收到本cpu ipi信号的其他cpu修改csd的flag
                for_each_cpu(cpu, cfd->cpumask) {
                        struct call_single_data *csd;

                        csd = per_cpu_ptr(cfd->csd, cpu);
                        csd_lock_wait(csd);
                }
        }
}

查看per_cpu_ptr的实现是通过将本cpu对应cfd->csd的值加上接收IPI信号的cpu号为偏移的数组__per_cpu_offset成员的值作为csd的地址值：

#ifndef __per_cpu_offset
extern unsigned long __per_cpu_offset[NR_CPUS];
#define per_cpu_offset(x) (__per_cpu_offset[x]) //从__per_cpu_offset数组获取percpu变量基地址，x为
#endif                                           // cpu号，本次问题x对应cpu1和cpu3
#ifndef SHIFT_PERCPU_PTR        
/* Weird cast keeps both GCC and sparse happy. */
#define SHIFT_PERCPU_PTR(__p, __offset) ({                              \
        __verify_pcpu_ptr((__p));                                       \
        RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
})
#endif

#ifdef CONFIG_SMP 
#define per_cpu_ptr(ptr, cpu)   SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
#else                   
#define per_cpu_ptr(ptr, cpu)   ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
#endif   

voidsmp_call_function_many(const struct cpumask *mask, smp_call_func_t func,void*info, bool wait)
{
    if (wait) {
                for_each_cpu(cpu, cfd->cpumask) {
                        struct call_single_data *csd;
                        //对应本次问题这里cfd->csd为cpu2的值，cpu为1和3
                        csd = per_cpu_ptr(cfd->csd, cpu);
                        csd_lock_wait(csd);
                }
        }

}

四核的机器，__per_cpu_offset只有前面四个数组成员有效：

crash> px __per_cpu_offset

__per_cpu_offset = $7 =

{0xffff88043fc00000, 0xffff88043fc80000, 0xffff88043fd00000, 0xffff88043fd80000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000, 0xffffffff81b2c000,...,...

}

或者用kmem -o查看per cpu 变量基准地址

crash> kmem -o

PER-CPU OFFSET VALUES:

CPU 0: ffff88043fc00000

CPU 1: ffff88043fc80000

CPU 2: ffff88043fd00000

CPU 3: ffff88043fd80000

crash>

从cpu2对应的per_cpu(cfd_data, 2)的cpumask知道，cpu2本次只给cpu1和cpu3发送了IPI信号，

而cpu2对应的cfd->csd值为0x1adb8

crash> p cfd_data:2

per_cpu(cfd_data, 2) = $8 = {

csd = 0x1adb8,

cpumask = 0xffff88017a1ee000

}

rash> struct cpumask 0xffff88017a1ee000

struct cpumask {

}

crash> eval -b 10

hexadecimal: a

decimal: 10

octal: 12

binary: 0000000000000000000000000000000000000000000000000000000000001010

bits set: 3 1

crash>

cpu1和cpu3的percpu变量基准地址分别为ffff88043fc80000 和 ffff88043fd80000

那么发送给cpu1的csd地址为0xffff88043fc9adb8

crash> px (0xffff88043fc80000+0x1adb8)

$11 = 0xffff88043fc9adb8

crash>

call_single_data .flags为0代表cpu1已经响应并处理了IPI信号：

crash> struct call_single_data 0xffff88043fc9adb8

struct call_single_data {

{

llist = {

next = 0x0

__UNIQUE_ID_rh_kabi_hide4 = {

list = {

next = 0x0,

prev = 0x0

}

{<No data fields>}

func = 0xffffffff8106e4a0 <flush_tlb_func>,

info = 0xffff88000399b830,

flags = 0

}

crash>

发送给cpu3的csd地址为0xffff88043fd9adb8

crash> px (0xffff88043fd80000+0x1adb8)

$12 = 0xffff88043fd9adb8

crash>

call_single_data .flags为1代表cpu3并未响应并处理了IPI信号：

crash> struct call_single_data 0xffff88043fd9adb8

struct call_single_data {

{

llist = {

next = 0xffff88043fd979c0

__UNIQUE_ID_rh_kabi_hide4 = {

list = {

next = 0xffff88043fd979c0,

prev = 0x0

}

{<No data fields>}

func = 0xffffffff8106e4a0 <flush_tlb_func>,

info = 0xffff88000399b830,

flags = 1

}

crash>

其实我们有更方便的方法可以找出死机时cpu2正在等待哪个csd的flags值被清零.

查看栈信息死机时对应RIP指令为smp_call_function_many+514：

crash> bt

PID: 12263 TASK: ffff8803bbebdee0 CPU: 2 COMMAND: "kworker/u8:0"

...

[exception RIP: smp_call_function_many+514]

RIP: ffffffff810f99a2 RSP: ffff88000399b7e8 RFLAGS: 00000202

RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff88043fd9adb8

对应的汇编代码如下

crash> dis smp_call_function_many

...

0xffffffff95916cb8 <smp_call_function_many+512>: pause

//call_single_data.flags值为1，所以testb两个操作数AND后的结果为1，RFLAGS的bit6标志位ZF应该为0，执行下面的jne导致一直在这死循环。当前flags为1，所以这里一直在jne和pause之间跳转。

0xffffffff95916cba <smp_call_function_many+514>: testb $0x1,0x20(%rcx)

0xffffffff95916cbe <smp_call_function_many+518>: jne 0xffffffff95916cb8 <smp_call_function_many+0x208>

....

...

crash> eval -b 0x202

hexadecimal: 202

decimal: 514

octal: 1002

binary: 0000000000000000000000000000000000000000000000000000001000000010

bits set: 9 1 //bit6 标志位ZF为0

crash>

smp_call_function_many+512到smp_call_function_many+518指令对应的c源码为csd_lock_wait函数：

static void csd_lock_wait(struct call_single_data *csd)

{

while (csd->flags & CSD_FLAG_LOCK)

cpu_relax();

}

csd的flags在call_single_data偏移0x20

crash> struct call_single_data.flags -xo

struct call_single_data {

[0x20] u16 flags;

}

crash>

testb $0x1,0x20(%rcx)指令对应的就是while (csd->flags & CSD_FLAG_LOCK)，那么rcx寄存器的值ffff88043fd9adb8自然就是call_single_data 类型csd的地址，ffff88043fd9adb8对应的是cpu3的csd地址，也就是代表cpu2正在等待cpu3响应IPI信号。

我们先来看看cpu在收到IPI信号后会做些什么？

static void flush_smp_call_function_queue(bool warn_cpu_offline)
{
   ...
//遍历本cpu对应call_single_queue队列中的csd，也就是处理发送给本cpu的所有ipi请求
   llist_for_each_entry_safe(csd, csd_next, entry, llist) {
                csd->func(csd->info);//ipi回调函数,本问题中对应的是flush_tlb_func
                csd_unlock(csd);//处理完回调函数后清除csd的flags
        }
    ...    
}

前面的分析我们已经知道cpu2是给cpu1和cpu3发送了ipi请求，而cpu1和cpu3对应的cpu_tlbstate.state

值都为2：

crash> p cpu_tlbstate

PER-CPU DATA TYPE:

struct tlb_state cpu_tlbstate;

PER-CPU ADDRESSES:

[0]: ffff88043fc16500

[1]: ffff88043fc96500

[2]: ffff88043fd16500

[3]: ffff88043fd96500

crash> struct tlb_state.state ffff88043fc96500

state = 2

crash> struct tlb_state.state ffff88043fd96500

state = 2

crash>

#define TLBSTATE_OK     1
#define TLBSTATE_LAZY   2
tatic void flush_tlb_func(void *info)
{       
        struct flush_tlb_info *f = info;
                
        inc_irq_stat(irq_tlb_count);

        if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
                return;

        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
                if (f->flush_end == TLB_FLUSH_ALL)
                        local_flush_tlb();
                else if (!f->flush_end)
                        __flush_tlb_single(f->flush_start);
                else {
                        unsigned long addr;
                        addr = f->flush_start;
                        while (addr < f->flush_end) {
                                __flush_tlb_single(addr);
                                addr += PAGE_SIZE;
                        }
                }
        } else
                leave_mm(smp_processor_id());//cpu_tlbstate.state为2，执行这里。

}

cpu_tlbstate.state不为TLBSTATE_OK时flush_tlb_func调用leave_mm(smp_processor_id())，

因为死机时CPU2发送IPI给cpu1和cpu3，那么这里smp_processor_id()值就是1和3,对应的

tlb_state.active_mm分别为 0xffff88042ce99900和0xffff88042cfb0640：

crash> struct tlb_state.active_mm ffff88043fc96500

active_mm = 0xffff88042ce99900

crash> struct tlb_state.active_mm ffff88043fd96500

active_mm = 0xffff88042cfb0640

crash>

void leave_mm(int cpu)  
{               
        struct mm_struct *active_mm = this_cpu_read(cpu_tlbstate.active_mm);
        if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
                BUG();  
        if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
                cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
                load_cr3(swapper_pg_dir);
        }                       
}

收到IPI请求的cpu处理完回调函数后会将cpu_tlbstate.active_mm的成员cpu_vm_mask_var

对应自己cpu号的位清零.

死机时cpu3对应cpu_tlbstate.active_mm的成员cpu_vm_mask_var的bit3位仍然为1

crash> struct mm_struct.cpu_vm_mask_var 0xffff88042cfb0640

cpu_vm_mask_var = 0xffff88042cfb0988

crash> rd 0xffff88042cfb0988

ffff88042cfb0988: 0000000000000008 ........

crash>

* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
{               
        return mm->cpu_vm_mask_var; 
}    
#define cpumask_bits(maskp) ((maskp)->bits)
/**     
 * cpumask_clear_cpu - clear a cpu in a cpumask
 * @cpu: cpu number (< nr_cpu_ids)
 * @dstp: the cpumask pointe
 */     
static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
{               
        clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
}

我们再回过头来看看触发死机的cpu2的调用栈

rash> bt

PID: 12263 TASK: ffff8803bbebdee0 CPU: 2 COMMAND: "kworker/u8:0"

....

[exception RIP: smp_call_function_many+514]

RIP: ffffffff810f99a2 RSP: ffff88000399b7e8 RFLAGS: 00000202

RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff88043fd9adb8

RDX: 0000000000000003 RSI: 0000000000000004 RDI: 0000000000000000

RBP: ffff88000399b820 R8: ffff88017a1ee000 R9: ffffffff813227d9

R10: ffff88043fd19c80 R11: ffffea00000c2100 R12: 0000000000000292

R13: ffff88000399b798 R14: ffffea0010b1f842 R15: 0000000000000000

ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018

#10 [ffff88000399b828] native_flush_tlb_others at ffffffff8106e668

#11 [ffff88000399b878] flush_tlb_page at ffffffff8106e864

#12 [ffff88000399b898] ptep_clear_flush at ffffffff811c2524

#13 [ffff88000399b8d0] page_mkclean at ffffffff811bbf2e

....

void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
        struct mm_struct *mm = vma->vm_mm;
    ...
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
                flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);             
        ...
}

static inline void flush_tlb_others(const struct cpumask *cpumask,
                                    struct mm_struct *mm,
                                    unsigned long start,
                                    unsigned long end)
{
        PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
}

void native_flush_tlb_others(const struct cpumask *cpumask,
                                 struct mm_struct *mm, unsigned long start,
                                 unsigned long end)
{
        ...
        smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
}

找出smp_call_function_many函数参数1 cpumask指针的地址

crash> whatis smp_call_function_many

void smp_call_function_many(const struct cpumask *, smp_call_func_t, void *, bool);

crash>

crash> dis -r ffffffff8106e668

...

0xffffffff8106e654 <native_flush_tlb_others+164>: mov $0x1,%ecx

0xffffffff8106e659 <native_flush_tlb_others+169>: mov $0xffffffff8106e4a0,%rsi

0xffffffff8106e660 <native_flush_tlb_others+176>: mov %r14,%rdi //r14和rdi存放的是smp_call_function_many函数参数1

0xffffffff8106e663 <native_flush_tlb_others+179>: callq 0xffffffff810f97a0 <smp_call_function_many>

0xffffffff8106e668 <native_flush_tlb_others+184>: jmp 0xffffffff8106e62a <native_flush_tlb_others+122>

crash> dis smp_call_function_many

0xffffffff810f97a0 <smp_call_function_many>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]

0xffffffff810f97a5 <smp_call_function_many+5>: push %rbp

0xffffffff810f97a6 <smp_call_function_many+6>: mov %rsp,%rbp

0xffffffff810f97a9 <smp_call_function_many+9>: push %r15

0xffffffff810f97ab <smp_call_function_many+11>: push %r14 //对r14进行压栈，此时r14存放的还是smp_call_function_many函数 //参数1

0xffffffff810f97ad <smp_call_function_many+13>: mov %rdx,%r14

0xffffffff810f97b0 <smp_call_function_many+16>: push %r13

0xffffffff810f97b2 <smp_call_function_many+18>: mov %rsi,%r13

0xffffffff810f97b5 <smp_call_function_many+21>: push %r12

0xffffffff810f97b7 <smp_call_function_many+23>: push %rbx

crash>bt -f

....

[exception RIP: smp_call_function_many+514]

RIP: ffffffff810f99a2 RSP: ffff88000399b7e8 RFLAGS: 00000202

RAX: 0000000000000003 RBX: 00000000000000fc RCX: ffff88043fd9adb8

RDX: 0000000000000003 RSI: 0000000000000004 RDI: 0000000000000000

RBP: ffff88000399b820 R8: ffff88017a1ee000 R9: ffffffff813227d9

R10: ffff88043fd19c80 R11: ffffea00000c2100 R12: 0000000000000292

R13: ffff88000399b798 R14: ffffea0010b1f842 R15: 0000000000000000

ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018

ffff88000399b740: ffffea0010b1f842 ffff88000399b798

ffff88000399b750: 0000000000000292 ffff88000399b820

ffff88000399b760: 00000000000000fc ffffea00000c2100

ffff88000399b770: ffff88043fd19c80 ffffffff813227d9

ffff88000399b780: ffff88017a1ee000 0000000000000003

ffff88000399b790: ffff88043fd9adb8 0000000000000003

ffff88000399b7a0: 0000000000000004 0000000000000000

ffff88000399b7b0: ffffffffffffff10 ffffffff810f99a2

ffff88000399b7c0: 0000000000000010 0000000000000202

ffff88000399b7d0: ffff88000399b7e8 0000000000000018

ffff88000399b7e0: ffffffff810f9977 000000012c7e1370

ffff88000399b7f0: ffff88042cfb0988 ffff88042cfb0640

ffff88000399b800: 00007f4474b04000 0000000000000000

ffff88000399b810: ffff88042cfb0988 ffff880114338820

ffff88000399b820: ffff88000399b870 ffffffff8106e668

#10 [ffff88000399b828] native_flush_tlb_others at ffffffff8106e668

....

通过栈信息找到smp_call_function_many函数参数1的地址为0xffff88042cfb0988

crash> struct cpumask 0xffff88042cfb0988

struct cpumask {

bits = {8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

}

crash>

cpu2的调用栈smp_call_function_many函数参数1的cpumask地址实际上跟前面分析的cpu3响应处理ipi请求后需要清除

其对应cpu掩码位的cpu_vm_mask_var是一个地址。

crash> struct mm_struct.cpu_vm_mask_var 0xffff88042cfb0640

cpu_vm_mask_var = 0xffff88042cfb0988

crash

所以实际上我们仅仅只要找到cpu2调用smp_call_function_many时传递的参数1 cpumask值就可以知道cpu2是在等待

哪个cpu响应其发送的ipi请求了。

smp_call_function_many函数第一个参数传入的cpumask对应地址的值代表

哪个cpu还未响应或正在处理IPI回调函数，也就是触发重启watchdog所在的cpu正在等待响应的cpu:

cfd_data:x的值来源于smp_call_function_many函数传入的参数1 cpumask，但是由于处理ipi请求的cpu在完成ipi请求后会清除smp_call_function_many函数传入的参数1 cpumask地址所保留的对应cpu位。所以cfd_data:x的cpumask代表死机前本cpu给哪几个cpu发送了IPI信号，而smp_call_function_many函数传入的参数1 cpumask地址的值代表本cpu正在等待哪个（些）cpu响应或者处理完成IPI请求，针对本问题对应信息如下：

cpu2给cpu1和cpu3发送了ipi请求

crash> p cfd_data:2

per_cpu(cfd_data, 2) = $15 = {

csd = 0x1adb8,

cpumask = 0xffff88017a1ee000

}

crash> struct cpumask 0xffff88017a1ee000

struct cpumask {

}

crash>

cpu2正在等待cpu3完成ipi请求，cpu1已经完成cpu2发送的ipi请求，所以bit1已被清0：

crash> struct cpumask 0xffff88042cfb0988

struct cpumask {

}

crash>

我们再来看看cpu3此时积压了多少ipi请求未处理

crash> p call_single_queue:3

per_cpu(call_single_queue, 3) = $19 = {

first = 0xffff88043fd9adb8

}

crash> list 0xffff88043fd9adb8

ffff88043fd9adb8 // cpu2发送过来的ipi请求

ffff88043fd979c0//

crash>

cpu3为什么不处理IPI请求？

cpu3此时运行着空闲swapper进程：

crash> bt -c 3

PID: 0 TASK: ffff88017a203f40 CPU: 3 COMMAND: "swapper/3"

bt: page excluded: kernel virtual address: ffffffffffffffff type: "cpu_online_map"

#0 [ffff88043fd85e48] crash_nmi_callback at ffffffff8104fd61

#1 [ffff88043fd85e58] nmi_handle at ffffffff816ad427

#2 [ffff88043fd85eb0] do_nmi at ffffffff816ad65d

#3 [ffff88043fd85ef0] end_repeat_nmi at ffffffff816ac8d3

[exception RIP: native_safe_halt+6]

RIP: ffffffff816ab4a6 RSP: ffff88017a23bea8 RFLAGS: 00000286

RAX: 00000000ffffffed RBX: ffffffff81b1c820 RCX: 0100000000000000

RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000046

RBP: ffff88017a23bea8 R8: 0000000000000000 R9: 0000000000000000

R10: 0000000000000000 R11: 0004ef1ee033ca80 R12: 0000000000000003

R13: ffff88017a238000 R14: ffff88017a238000 R15: ffff88017a238000

ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018

--- <NMI exception stack> ---

#4 [ffff88017a23bea8] native_safe_halt at ffffffff816ab4a6

#5 [ffff88017a23beb0] default_idle at ffffffff816ab33e

#6 [ffff88017a23bed0] arch_cpu_idle at ffffffff81035006

#7 [ffff88017a23bee0] cpu_startup_entry at ffffffff810e7bca

#8 [ffff88017a23bf28] start_secondary at ffffffff81051af6

crash>

cpu3运行队列里却是有进程在等cpu资源：

crash> runq

...

CPU 3 RUNQUEUE: ffff88043fd96cc0

CURRENT: PID: 0 TASK: ffff88017a203f40 COMMAND: "swapper/3"

RT PRIO_ARRAY: ffff88043fd96e50

[no tasks queued]

CFS RB_ROOT: ffff88043fd96d68

[120] PID: 30118 TASK: ffff880012ab8000 COMMAND: "barad_agent"

[120] PID: 30121 TASK: ffff8800368dbf40 COMMAND: "java"

crash>

cpu3并未出现死锁，但是却没有响应ipi请求和调度运行可运行队列中的进程，由于是kvm虚拟机，因此推测是host主机出现异常导致子机vcpu3没有获取cpu资源导致的，排查host主机日志后发现是由于磁盘故障引起该子机对应的qemu进程D状态引起的。

原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。

如有侵权，请联系 cloudcommunity@tencent.com 删除。

tcp/ip

原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。

如有侵权，请联系 cloudcommunity@tencent.com 删除。

tcp/ip

从一个softlock问题来谈谈Kernel IPI的实现

从一个softlock问题来谈谈Kernel IPI的实现

社区

活动

资源

关于

腾讯云开发者

热门产品

热门推荐

更多推荐