
ls -lt /proc/<pid>/ns
# ls /proc/2019/ns -lt
总用量 0
lrwxrwxrwx 1 root root 0 8月 29 09:45 pid_for_children -> pid:[4026533202]
lrwxrwxrwx 1 root root 0 8月 29 09:43 cgroup -> cgroup:[4026531835]
lrwxrwxrwx 1 root root 0 8月 29 09:43 ipc -> ipc:[4026533201]
lrwxrwxrwx 1 root root 0 8月 29 09:43 net -> net:[4026533204]
lrwxrwxrwx 1 root root 0 8月 29 09:43 pid -> pid:[4026533202]
lrwxrwxrwx 1 root root 0 8月 29 09:37 uts -> uts:[4026533200]
lrwxrwxrwx 1 root root 0 8月 29 09:37 user -> user:[4026531837]
lrwxrwxrwx 1 root root 0 8月 29 09:37 mnt -> mnt:[4026533199]解释如下:
net -> net:4026533204
net:4026533204是符号链接,指向内核维护的namespace句柄。
4026533204为net namespace 编号。相同编号的进程,位于同一namespace。
使用nsenter进入docker的namespace,这样就可以用传统命令排查故障了。
(1) help
nsenter的help如下。
# nsenter -h
用法:
nsenter [options] <program> [<argument>...]
Run a program with namespaces of other processes.
选项:
-t, --target <pid> 要获取名字空间的目标进程
-m, --mount[=<file>] enter mount namespace
-u, --uts[=<file>] enter UTS namespace (hostname etc)
-i, --ipc[=<file>] enter System V IPC namespace
-n, --net[=<file>] enter network namespace
-p, --pid[=<file>] enter pid namespace
-U, --user[=<file>] enter user namespace
-S, --setuid <uid> set uid in entered namespace
-G, --setgid <gid> set gid in entered namespace
--preserve-credentials do not touch uids or gids
-r, --root[=<dir>] set the root directory
-w, --wd[=<dir>] set the working directory
-F, --no-fork 执行 <程序> 前不 fork
-Z, --follow-context set SELinux context according to --target PID
-h, --help 显示此帮助并退出
-V, --version 输出版本信息并退出
更多信息请参阅 nsenter(1)。(2) 即时命令还是交互式
nsenter命令后面如果不跟program,那么就会进入相应的namespace,当前shell变成对应的namespace的shell,exit可以退出。
例如:nsenter -t <pid> -n 直接回车,切换到对应的netns。
(3) pid问题
主要是pid得拿到正确的,有一些是docker自己的。
/usr/bin/dockerd
docker-containerd -> docker-proxy
docker-containerd-shim -> docker容器内部命令(譬如pmm-server为/usr/bin/python /usr/bin/supervisord -n -c /etc/supervisord.conf)pid要用docker容器内部命令及其子进程的。
进入network namespace,这样就可以使用netstat等宿主机的命令来排查docker的网络问题。
nenter -t <pid> -n <program>
-n参数,表示进入这个pid的network namespace。
-t参数,表示目标进程的PID。例如:
# nsenter -t 2019 -n ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
14: eth0@if15: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
link/ether 02:42:ac:11:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 172.17.0.2/16 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::42:acff:fe11:2/64 scope link
valid_lft forever preferred_lft forever
# nsenter -t 2019 -n netstat -ltnp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:3306 0.0.0.0:* LISTEN 2270/mysqld
tcp 0 0 127.0.0.1:7777 0.0.0.0:* LISTEN 2284/pmm-configurat
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 2273/nginx: master
tcp 0 0 0.0.0.0:443 0.0.0.0:* LISTEN 2273/nginx: master
tcp 0 0 127.0.0.1:9100 0.0.0.0:* LISTEN 2283/node_exporter
tcp6 0 0 :::3000 :::* LISTEN 2272/grafana-server
tcp6 0 0 :::4000 :::* LISTEN 8001/orchestrator
tcp6 0 0 :::8300 :::* LISTEN 2271/consul
tcp6 0 0 :::8301 :::* LISTEN 2271/consul
tcp6 0 0 :::8302 :::* LISTEN 2271/consul
tcp6 0 0 :::8500 :::* LISTEN 2271/consul
tcp6 0 0 :::8600 :::* LISTEN 2271/consul
tcp6 0 0 :::9001 :::* LISTEN 6140/percona-qan-ap
tcp6 0 0 :::9090 :::* LISTEN 2276/prometheus查看docker容器内的所有进程。只用-p参数不行,只用-m也不行。得2个一起。
nsenter -t 2019 -m -p ps -ef以上也可以直接只用-a参数(nsenter,来自 util-linux 2.32.1,有这个参数,早期版本似乎没有),进入该docker容器的所有namespace
例如,查看docker容器内的所有进程。
# nsenter -t 2019 -a ps -ef
UID PID PPID C STIME TTY TIME CMD
root 1 0 0 May15 ? 01:11:01 /usr/bin/python /usr/bin/supervisord -n -c /etc/supervisord.conf
mysql 17 1 0 May15 ? 07:22:33 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib64/mysql/plugin --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock
pmm 18 1 0 May15 ? 04:32:42 /usr/sbin/consul agent -server -data-dir=/opt/consul-data -bootstrap -client=0.0.0.0 -advertise=127.0.0.1 -ui
grafana 19 1 0 May15 ? 01:13:07 /usr/sbin/grafana-server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.paths.plugins=/var/lib/grafana/plugins cfg
root 20 1 0 May15 ? 00:00:00 nginx: master process nginx
root 21 1 0 May15 ? 00:00:06 /usr/sbin/crond -n
pmm 22 1 0 May15 ? 00:00:00 bash -c sleep 5 && /usr/sbin/percona-qan-api -srcPath /usr/share/percona-qan-api/src -importPath github.com/percona/qan-api -runMode prod
pmm 23 1 3 May15 ? 3-15:10:12 /usr/sbin/prometheus -config.file=/etc/prometheus.yml -storage.local.path=/opt/prometheus/data -web.listen-address=:9090 -storage.local.retention=720h -storage.local.memory-chunks=262144 -storage.local.chunk-encoding-vers
pmm 25 1 0 May15 ? 00:00:00 bash -c sleep 7 && /usr/sbin/orchestrator http
root 30 1 2 May15 ? 2-12:51:07 /usr/sbin/node_exporter -web.listen-address=localhost:9100 -collectors.enabled=diskstats,filefd,filesystem,loadavg,meminfo,netdev,netstat,stat,time,uname,vmstat
root 31 1 0 May15 ? 00:00:03 /usr/sbin/pmm-configurator -ssh-key-owner pmm -grafana-db-path /var/lib/grafana/grafana.db
nginx 134 20 0 May15 ? 00:04:04 nginx: worker process
nginx 135 20 0 May15 ? 00:04:03 nginx: worker process
nginx 136 20 0 May15 ? 00:04:02 nginx: worker process
nginx 137 20 0 May15 ? 00:04:02 nginx: worker process
nginx 138 20 0 May15 ? 00:04:01 nginx: worker process
nginx 139 20 0 May15 ? 00:04:00 nginx: worker process
nginx 140 20 0 May15 ? 00:04:03 nginx: worker process
nginx 141 20 0 May15 ? 00:04:06 nginx: worker process
nginx 142 20 0 May15 ? 00:04:05 nginx: worker process
nginx 143 20 0 May15 ? 00:04:08 nginx: worker process
nginx 144 20 0 May15 ? 00:00:00 nginx: worker process
nginx 145 20 0 May15 ? 00:04:06 nginx: worker process
nginx 146 20 0 May15 ? 00:04:05 nginx: worker process
nginx 147 20 0 May15 ? 00:04:05 nginx: worker process
nginx 148 20 0 May15 ? 00:04:04 nginx: worker process
nginx 149 20 0 May15 ? 00:04:05 nginx: worker process
nginx 150 20 0 May15 ? 00:04:02 nginx: worker process
nginx 151 20 0 May15 ? 00:04:04 nginx: worker process
nginx 152 20 0 May15 ? 00:04:05 nginx: worker process
nginx 153 20 0 May15 ? 00:04:03 nginx: worker process
nginx 154 20 0 May15 ? 00:04:03 nginx: worker process
nginx 155 20 0 May15 ? 00:04:03 nginx: worker process
nginx 158 20 0 May15 ? 00:04:03 nginx: worker process
nginx 160 20 0 May15 ? 00:04:03 nginx: worker process
nginx 161 20 0 May15 ? 00:04:04 nginx: worker process
nginx 162 20 0 May15 ? 00:04:01 nginx: worker process
nginx 163 20 0 May15 ? 00:04:05 nginx: worker process
nginx 164 20 0 May15 ? 00:04:07 nginx: worker process
nginx 165 20 0 May15 ? 00:04:06 nginx: worker process
nginx 166 20 0 May15 ? 00:04:04 nginx: worker process
nginx 167 20 0 May15 ? 00:04:07 nginx: worker process
nginx 168 20 0 May15 ? 00:04:07 nginx: worker process
nginx 169 20 0 May15 ? 00:04:08 nginx: worker process
nginx 170 20 0 May15 ? 00:04:07 nginx: worker process
nginx 171 20 0 May15 ? 00:04:05 nginx: worker process
nginx 172 20 0 May15 ? 00:04:09 nginx: worker process
nginx 173 20 0 May15 ? 00:04:13 nginx: worker process
nginx 174 20 0 May15 ? 00:04:06 nginx: worker process
nginx 175 20 0 May15 ? 00:04:08 nginx: worker process
nginx 176 20 0 May15 ? 00:04:08 nginx: worker process
pmm 200 22 0 May15 ? 01:20:04 /usr/sbin/percona-qan-api -srcPath /usr/share/percona-qan-api/src -importPath github.com/percona/qan-api -runMode prod
pmm 212 200 0 May15 ? 00:00:00 perl /usr/share/percona-qan-api/src/github.com/percona/qan-api/service/query/mini.pl
pmm 216 25 0 May15 ? 04:24:34 /usr/sbin/orchestrator http
root 31688 0 0 09:10 ? 00:00:00 ps -efdocker的veth关系查找
进入容器获取 eth0 的 iflink 值
docker exec <容器ID> cat /sys/class/net/eth0/iflink在宿主机根据 iflink 值查找对应 veth 接口
grep -l <iflink值> /sys/class/net/veth*/ifindex用nsenter查找
查看docker容器内的接口信息
# nsenter -t 2019 -n ip -d link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 addrgenmode eui64
14: eth0@if15: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT
link/ether 02:42:ac:11:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
veth addrgenmode eui64或者查看docker容器内的接口的index和对端
# nsenter -t 2019 -a cat /sys/class/net/eth0/ifindex
14
# nsenter -t 2019 -a cat /sys/class/net/eth0/iflink
15用第1个接口信息就够了,就知道对端是index为15的接口。到宿主机上查看
grep -l <iflink值> /sys/class/net/veth*/ifindex原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。