一次线上服务高 CPU 占用优化实践

top -Hp you_pid

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
   48 root      20   0 30.367g 2.636g  12940 S  12.7  2.9  36:15.18 java
 2365 root      20   0 30.367g 2.636g  12940 R  1.3  2.9   2:33.64 java
 2380 root      20   0 30.367g 2.636g  12940 S  1.3  2.9   2:33.10 java
 2381 root      20   0 30.367g 2.636g  12940 S  1.3  2.9   2:33.41 java
10079 root      20   0 30.367g 2.636g  12940 S  1.3  2.9   0:30.73 java
   10 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   4:08.54 java
   11 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   4:08.55 java
   92 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   2:53.71 java
  681 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   2:52.56 java
  683 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   2:56.81 java
  690 root      20   0 30.367g 2.636g  12940 S  1.0  2.9   3:34.24 java

jstack 1 | grep -A 10 "0x30 "

"kafka-producer-network-thread | producer-1" #35 daemon prio=5 os_prio=0 tid=0x00007f9ac4fc7000 nid=0x30 runnable [0x00007f9ac9b88000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        - locked <0x0000000094ef70c8> (a sun.nio.ch.Util$3)
        - locked <0x0000000094ef70e0> (a java.util.Collections$UnmodifiableSet)
        - locked <0x000000009642bbb8> (a sun.nio.ch.EPollSelectorImpl)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.kafka.common.network.Selector.select(Selector.java:686)

<mvc:interceptors>
    <mvc:interceptor>
        <mvc:mapping path="/**"/>
        <mvc:exclude-mapping path="/login"/>
        <mvc:exclude-mapping path="/*/login"/>
        <mvc:exclude-mapping path="/*/activity/teacher"/>
        <mvc:exclude-mapping path="/masaike/dynamic/**"/>
        ...下面还有一百多个这样的 exclude...
        <mvc:exclude-mapping path="/masaike/aaaa/**"/>
        <mvc:exclude-mapping path="/masaike/**/hello"/>
       <bean class="com.masaike.AuthenticationHandlerInterceptor"/>
    </mvc:interceptor>
</mvc:interceptors>

  "/api/*/login"
, "/wildcard/**"
, "/wildcard/**/hello"
, "/v2/hello/"
, "/v2/user/info/"
, "/v2/user/feed/"
, "/v2/user/feed2/"

.
└──api
	└──*
		└──login
└──v2
	└──hello
	└──user
		└──feed
		└──feed2
		└──info
└──wildcard
	└──**

/**
 * @author Arthur.Zhang (zhangya@imlizhi.com)
 * 用于匹配 /a/b/c, *, ** 这种格式的前缀匹配
 * 参考了 ZooKeeper 和 lucene 的 trie 实现
 */
public class PathTrie {
    private Node root = new Node(emptyList(), null);

    public PathTrie() {
    }

    public void insert(String path, String obj) {
        List<String> parts = getPathSegments(path);
        insert(parts, obj);
    }

    public void insert(List<String> parts, String o) {
        if (parts.isEmpty()) {
            root.obj = o;
            return;
        }
        root.insert(parts, o);
    }

    private static List<String> getPathSegments(String path) {
        return Splitter.on('/').splitToList(path).stream().filter(it -> !it.isEmpty()).collect(Collectors.toList());
    }

    public boolean existsPath(String path) {
        return root.exists(getPathSegments(path), 0);
    }

    public void dump() {
        if (root != null) root.dump();
    }

    private static class Node {
        String name;
        Map<String, Node> children;
        String obj; // 标识是否是叶子节点

        Node(List<String> path, String obj) {
            if (path.isEmpty()) {
                this.obj = obj;
                return;
            }
            name = path.get(0);
        }

        private synchronized void insert(List<String> parts, String o) {
            String part = parts.get(0);
            Node matchedChild;

            // 如果是 **，后面的路径节点不用再插入了
            if ("**".equals(name)) {
                return;
            }
            if (children == null) children = new ConcurrentHashMap<>();

            matchedChild = children.get(part);
            if (matchedChild == null) {
                matchedChild = new Node(parts, o);
                children.put(part, matchedChild);
            }

            // 移除已处理的
            parts.remove(0);
            if (parts.isEmpty()) { // 如果已经到底，将最后一个 child 的 obj 赋值
                matchedChild.obj = o;
            } else {
                matchedChild.insert(parts, o); //还没有到底，继续递归插入
            }

        }

        /**
         * @param pathSegments 路径分割以后的 word 列表，比如 /a/b/c -> 'a' , 'b' , 'c'
         * @param level        当前路径遍历的 level 深度，比如 /a/b/c -> 0='a' 1='b' 2='c'
         */
        public boolean exists(List<String> pathSegments, int level) {
            // 如果已经把传入的 path 遍历完了
            if (pathSegments.size() < level + 1) {
                // 如果当前 trie 树不是叶子节点
                if (obj == null) {
                    // 获取叶子节点是否包含 **，如果包含的话，则匹配到
                    Node n = children.get("**");
                    if (n != null) {
                        return true;
                    }
                }
                return obj != null;
            }
            if (children == null) {
                return false;
            }
            String pathSegment = pathSegments.get(level);

            // 1、首先找绝对匹配的
            Node n = children.get(pathSegment);
            // 2、如果不存在，则找是否包含 * 的
            if (n == null) {
                n = children.get("*");
            }
            // 3、如果还不存在，则找是否包含 ** 的
            if (n == null) {
                n = children.get("**");
                if (n != null) {
                    return true;
                }
            }
            // 4、如果这些都没有找到，则返回 false
            if (n == null) {
                return false;
            }
            // 5、如果找到了一个 node，则继续递归查找
            return n.exists(pathSegments, level + 1);
        }

        @Override
        public String toString() {
            return "Node{" +
                           "name='" + name + '\'' +
                           ", children=" + children +
                           '}';
        }

        /**
         * 使用类似 tree 命令的输出格式打印这棵前缀数
         * .
         * └──api
         * 	└──*
         * 		└──login
         * └──v2
         * 	└──hello
         * 	└──user
         * 		└──feed
         * 		└──feed2
         * 		└──info
         * └──wildcard
         * 	    └──**
         */
        public void dump() {
            dump(0);
        }

        public void dump(int level) {
            if (level == 0) {
                System.out.println(".");
            } else {
                StringBuilder sb = new StringBuilder();
                for (int i = 0; i < level - 1; i++) {
                    sb.append("\t");
                }
                sb.append("└──").append(name);
                System.out.println(sb.toString());
            }
            if (children != null && !children.isEmpty()) {
                for (Map.Entry<String, Node> item : children.entrySet()) {
                    item.getValue().dump(level + 1);
                }
            }
        }
    }
}