kernel rootkits引发的系统重启

查看栈信息内核是由于访问了非法地址ffff9d713fffffff触发了异常重启：

rdi寄存器的值是在上一层调用传入的值，当前无法从strnstr看出rdi地址的来源.

结合栈信息以及源码可以找到seq_read是在调用err = m->op->show(m, p)进入的strnstr:

crash> dis -rl ffffffff9da41410

0xffffffff9da4140b <seq_read+267>: callq 0xffffffff9db5a550 <__x86_indirect_thunk_rax>

/usr/src/debug/kernel-3.10.0-862.el7/linux-3.10.0-862.el7.x86_64/fs/seq_file.c: 241

0xffffffff9da41410 <seq_read+272>: test %eax,%eax

ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos)

{

struct seq_file *m = file->private_data;

....

....

while (1) {

.....

......

err = m->op->show(m, p);

.....

....

}

....

....

}

接着需要找出m->op->show对应的地址:

结合源码可知只需要找到struct file *file 也就可以找出m->op->show了，那么怎么找出struct file *file呢？

crash> whatis proc_reg_read

ssize_t proc_reg_read(struct file *, char *, size_t, loff_t *);

crash>

crash> dis -r ffffffff9da1ab3f

0xffffffff9da1ab20 <vfs_read+128>: mov 0x28(%rbx),%rax

0xffffffff9da1ab24 <vfs_read+132>: mov %r13,%rcx

0xffffffff9da1ab27 <vfs_read+135>: mov %r12,%rsi

//rbx和rdi存放vfs_read调用proc_reg_read时传递的第一个参数：

0xffffffff9da1ab2a <vfs_read+138>: mov %rbx,%rdi

0xffffffff9da1ab2d <vfs_read+141>: mov 0x10(%rax),%rax

0xffffffff9da1ab31 <vfs_read+145>: test %rax,%rax

0xffffffff9da1ab34 <vfs_read+148>: je 0xffffffff9da1abe0 <vfs_read+320>

//这里对应调用proc_reg_read

0xffffffff9da1ab3a <vfs_read+154>: callq 0xffffffff9db5a550 <__x86_indirect_thunk_rax>

0xffffffff9da1ab3f <vfs_read+159>: mov %rax,%r12

crash> dis proc_reg_read

0xffffffff9da90670 <proc_reg_read>: nopl 0x0(%rax,%rax,1) [FTRACE NOP]

0xffffffff9da90675 <proc_reg_read+5>: push %rbp

0xffffffff9da90676 <proc_reg_read+6>: xor %r8d,%r8d

0xffffffff9da90679 <proc_reg_read+9>: mov %rsp,%rbp

//proc_reg_read函数中对rbx进行了压栈

0xffffffff9da9067c <proc_reg_read+12>: push %rbx

0xffffffff9da9067d <proc_reg_read+13>: sub $0x8,%rsp

0xffffffff9da90681 <proc_reg_read+17>: mov 0x20(%rdi),%rax

0xffffffff9da90685 <proc_reg_read+21>: mov -0x28(%rax),%rbx

找出栈中rbx的值，也就是相当于找到了struct file *file地址：

crash> bt -f

...

...

#11 [ffff9d744ce13eb8] proc_reg_read at ffffffff9da906b0

ffff9d744ce13ec0: 00007f36d7693000 ffff9d736aab9200

ffff9d744ce13ed0: ffff9d744ce13f00 ffffffff9da1ab3f

#12 [ffff9d744ce13ed8] vfs_read at ffffffff9da1ab3f

ffff9d744ce13ee0: ffff9d736aab9200 0000000000000003

ffff9d744ce13ef0: 00007f36d7693000 0000000000000400

ffff9d744ce13f00: ffff9d744ce13f48 ffffffff9da1ba0f

#13 [ffff9d744ce13f08] sys_read at ffffffff9da1ba0f

crash> struct file.private_data ffff9d736aab9200

private_data = 0xffff9d71fb70f6c0

crash>

0xffff9d71fb70f6c0为类型struct seq_file 地址

crash> struct seq_file.op 0xffff9d71fb70f6c0

op = 0xffffffff9e5098d8

crash> struct seq_file.op -xo

struct seq_file {

[0x60] const struct seq_operations *op;

}

crash>

crash> struct seq_operations 0xffffffff9e5098d8

struct seq_operations {

start = 0xffffffff9de55990,

stop = 0xffffffff9de54a90,

next = 0xffffffff9de55900,

show = 0xffffffffc0569210

}

crash>

这里的show = 0xffffffffc0569210就是前面函数seq_read

对应的m->op->show。

查看struct seq_operations 0xffffffff9e5098d8其他几个成员（start/stop/next）都是正常的对应内核的函数，只有show是一个未知的地址：

crash> struct tcp_seq_afinfo -xo

struct tcp_seq_afinfo {

[0x0] char *name;

[0x8] sa_family_t family;

[0x10] const struct file_operations *seq_fops;

[0x18] struct seq_operations seq_ops;

}

SIZE: 0x38

crash>

seq_operations在tcp_seq_afinfo 偏移0x18位置，所以对应的tcp_seq_afinfo

地址为0xffffffff9e5098c0：

crash> px 0xffffffff9e5098d8-0x18

$16 = 0xffffffff9e5098c0

crash>

ffffffff9e5098c0是tcp6_seq_afinfo地址：

crash> sym ffffffff9e5098c0

ffffffff9e5098c0 (d) tcp6_seq_afinfo

crash> px tcp6_seq_afinfo

tcp6_seq_afinfo = $17 = {

name = 0xffffffff9e2ecc6c "tcp6",

family = 0xa,

seq_fops = 0xffffffff9e0ad420,

seq_ops = {

start = 0xffffffff9de55990,

stop = 0xffffffff9de54a90,

next = 0xffffffff9de55900,

show = 0xffffffffc0569210

}

}

crash>

因此理论上m->op->show对应的应该是tcp6_seq_show，可是在机器panic时实际地址是0xffffffffc0569210。

crash> sym tcp6_seq_show

ffffffff9ded4010 (t) tcp6_seq_show /usr/src/debug/kernel-3.10.0-862.el7/linux-3.10.0-862.el7.x86_64/net/ipv6/tcp_ipv6.c: 1785

crash>

从0xffffffffc0569210可以知道这是一个模块地址，当时系统死机时并未加载第三方模块：

crash> mod -t

no tainted modules

crash>

把内存地址0xffffffffc0569210 的值读取出来：

crash> rd -8 0xffffffffc0569210 1024 | awk -F " " '{for (i=2;i<=NF-1;i++)printf("%s ", $i)}'

0f 1f 44 00 00 55 48 89 e5 41 56 41 55 45 31 ed 41 54 45 31 e4 53 48 89 fb 48 83 ec 10 65 48 8b 04 25 28 00 00 00 48 89 45 d8 31 c0 48 8b 05 05 25 00 00 e8 08 13 5f dd 41 89 c6 eb 19 0f 1f 00 49 83 c4 01 48 81 6b 18 b4 00 00 00 49 83 fc 0b 0f 84 8a 00 00 00 42 8b 0c a5 a0 b4 56 c0 48 8d 7d d2 48 c7 c2 24 a0 56 c0 be 06 00 00 00 31 c0 e8 6b e6 5e dd 48 8b 13 48 8b 43 18 48 8d 75 d2 48 8d bc 02 4c ff ff ff ba b4 00 00 00 e8 9e b0 5e dd 48 85 c0 75 a9 41 83 fc 04 44 89 e8 48 8b 13 41 0f 4e c4 48 98 48 8d 34 c5 60 b4 56 c0 48 8b 43 18 48 8d bc 02 4c ff ff ff ba b4 00 00 00 e8 6b b0 5e dd 48 85 c0 0f 85 72 ff ff ff 49 83 c4 01 49 83 fc 0b 0f 85 7a ff ff ff 0f 1f 40 00 48 8b 4d d8 65 48 33 0c 25 28 00 00 00 44 89 f0 75 0d 48 83 c4 10 5b 41 5c 41 5d 41 5e 5d c3 e8 1c 81 32 dd 66 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 56 41 55 45 31 ed 41 54 45 31 e4 53 48 89 fb 48 83 ec 10 65 48 8b 04 25 28 00 00 00 48 89 45 d8 31 c0 48 8b 05 ed 23 00 00 e8 f8 11 5f dd 41 89 c6 eb 19 0f 1f 00 49 83 c4 01 48 81 6b 18 a8 00 00 00 49 83 fc 0b 0f 84 8a 00 00 00 42 8b 0c a5 a0 b4 56 c0 48 8d 7d d2 48 c7 c2 24 a0 56 c0 be 06 00 00 00 31 c0 e8 5b e5 5e dd 48 8b 13 48 8b 43 18 48 8d 75 d2 48 8d bc 02 58 ff ff ff ba a8 00 00 00 e8 8e af 5e dd 48 85 c0 75 a9 41 83 fc 04 44 89 e8 48 8b 13 41 0f 4e c4 48 98 48 8d 34 c5 60 b4 56 c0 48 8b 43 18 48 8d bc 02 58 ff ff ff ba a8 00 00 00 e8 5b af 5e dd 48 85 c0 0f 85 72 ff ff ff 49 83 c4 01 49 83 fc 0b 0f 85 7a ff ff ff 0f 1f 40 00 48 8b 4d d8 65 48 33 0c 25 28 00 00 00 44 89 f0 75 0d 48 83 c4 10 5b 41 5c 41 5d 41 5e 5d c3 e8 0c 80 32 dd 66 90 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 65 48 8b 34 25 40 0e 01 00 48 8b 96 30 04 00 00 48 89 e5 48 8d 82 d0 fb ff ff 48 39 f0 74 30 39 7a 74 74 2d b9 d0 07 00 00 eb 11 0f 1f 80 00 00 00 00 39 7a 74 74 1a 83 e9 01 74 13 48 8b 90 30 04 00 00 48 8d 82 d0 fb ff ff 48 39 f0 75 e3 31 c0 5d c3 0f 1f 80 00 00 00 00 0f 1f 44 00 00 65 48 8b 34 25 40 0e 01 00 48 8b 96 30 04 00 00 48 8d 82 d0 fb ff ff 48 39 c6 74 2c 3b 7a 74 74 2a b9 d0 07 00 00 eb 0d 0f 1f 00 3b 7a 74 74 1b 83 e9 01 74 13 48 8b 90 30 04 00 00 48 8d 82 d0 fb ff ff 48 39 c6 75 e3 31 c0 c3 48 85 c0 74 f8 55 48 89 e5 41 54 4c 8d a0 78 06 00 00 53 48 c7 c3 40 b1 56 c0 eb 11 0f 1f 40 00 48 83 c3 10 48 81 fb 50 b4 56 c0 74 23 48 89 de 4c 89 e7 e8 98 ad 5e dd 48 85 c0 74 e3 5b 41 5c b8 01 00 00 00 5d c3 66 0f 1f 84 00 00 00 00 00 5b 41 5c 31 c0 5d c3 66 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 53 48 89 fb e8 9e a9 5e dd 48 8d 54 03 ff 48 39 d3 77 4c 0f be 32 b8 ff ff ff ff 8d 4e d0 80 f9 09 77 2f b9 01 00 00 00 31 c0 eb 10 0f 1f 00 0f be 32 44 8d 46 d0 41 80 f8 09 77 1b 83 ee 30 48 83 ea 01 0f af f1 8d 0c 89 01 c9 01 f0 48 39 d3 76 dd 5b 5d c3 66 90 5b b8 ff ff ff ff 5d c3 31 c0 eb ef 0f 1f 40 00 0f 1f 44 00 00 55 48 89 e5 41 57 41 56 41 55 41 54 53 48 89 f3 48 83 ec 10 48 89 75 c8 48 8b 05 64 21 00 00 e8 77 0f 5f dd be d0 00 00 00 49 89 c5 48 89 c7 e8 87 eb 48 dd 44 89 ea 48 89 de 48 89 c7 49 89 c7 48 89 45 d0 e8 62 f5 5e dd 85 c0 0f 85 9b 00 00 00 45 85 ed 45 89 ec 44 89 eb 7e crash>

保存到oop.txt文件中，文件开头需要加Code:

利用内核源码中的scripts/decodecode将机器码转为汇编语言,https://onlinedisassembler.com/odaweb/在线工具也可以将机器码转为汇编：

# ./scripts/decodecode < ./oop.txt

Code: 0f 1f 44 00 00 55 48 89 e5 41 56 41 55 45 31 ed 41 54 45 31 e4 53 48 89 fb 48 83 ec 10 65 48 8b 04 25 28 00 00 00 48 89 45 d8 31 c0 48 8b 05 05 25 00 00 e8 08 13 5f dd 41 89 c6 eb 19 0f 1f 00 49 83 c4 01 48 81 6b 18 b4 00 00 00 49 83 fc 0b 0f

.....

....

Code starting with the faulting instruction

===========================================

0: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)

5: 55 push %rbp

6: 48 89 e5 mov %rsp,%rbp

9: 41 56 push %r14

b: 41 55 push %r13

d: 45 31 ed xor %r13d,%r13d

...

....

56: 42 8b 0c a5 a0 b4 56 mov -0x3fa94b60(,%r12,4),%ecx

5d: c0

5e: 48 8d 7d d2 lea -0x2e(%rbp),%rdi

62: 48 c7 c2 24 a0 56 c0 mov $0xffffffffc056a024,%rdx

...

...

读取0xffffffffc056a024地址内容可以发现如下信息，疑似系统调用SyS_getdents被修改：

系统调用表确实被修改了

启动日志中也确实看到内核启动的时候加载过iproute模块：

crash> log | grep tainting

[ 8.929993] iproute: module verification failed: signature and/or required key missing - tainting kernel

crash>

因此推测内核是中了skidmap病毒：

https://www.technadu.com/skidmap-new-cryptomining-malware-linux-systems/80243/

https://www.virustotal.com/gui/home/search

内存中看到有pamdicks字符串，顺藤摸瓜也就找到了/usr/bin/pamdicks病毒文件：

如何杀毒防毒这里就不深入表述了，中毒机器条件允许还是直接重装来得干净彻底！！