浅谈 URL 解析与鉴权中的陷阱

Client -> Filter_1 -> Filter_2 -> ... -> Filter_n -> Servlet

@WebServlet(name = "flagServlet", value = "/api/flag")
public class FlagServlet extends HttpServlet {
    public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
        PrintWriter out = response.getWriter();
        out.println(String.format("YOU WIN: [%s]", request.getRequestURI()));
    }
}

doGet:17, FlagServlet (com.example.tc)
service:529, HttpServlet (javax.servlet.http)
service:623, HttpServlet (javax.servlet.http)
internalDoFilter:209, ApplicationFilterChain (org.apache.catalina.core)
doFilter:153, ApplicationFilterChain (org.apache.catalina.core)
doFilter:51, WsFilter (org.apache.tomcat.websocket.server)
internalDoFilter:178, ApplicationFilterChain (org.apache.catalina.core)
doFilter:153, ApplicationFilterChain (org.apache.catalina.core)
doFilter:20, AdminFilter (com.example.tc)
doFilter:53, HttpFilter (javax.servlet.http)
internalDoFilter:178, ApplicationFilterChain (org.apache.catalina.core)
doFilter:153, ApplicationFilterChain (org.apache.catalina.core)
invoke:167, StandardWrapperValve (org.apache.catalina.core)
invoke:90, StandardContextValve (org.apache.catalina.core)
invoke:481, AuthenticatorBase (org.apache.catalina.authenticator)
invoke:130, StandardHostValve (org.apache.catalina.core)
invoke:93, ErrorReportValve (org.apache.catalina.valves)
invoke:673, AbstractAccessLogValve (org.apache.catalina.valves)
invoke:74, StandardEngineValve (org.apache.catalina.core)
service:343, CoyoteAdapter (org.apache.catalina.connector)
service:390, Http11Processor (org.apache.coyote.http11)
process:63, AbstractProcessorLight (org.apache.coyote)
process:926, AbstractProtocol$ConnectionHandler (org.apache.coyote)
doRun:1791, NioEndpoint$SocketProcessor (org.apache.tomcat.util.net)
run:52, SocketProcessorBase (org.apache.tomcat.util.net)
runWorker:1191, ThreadPoolExecutor (org.apache.tomcat.util.threads)
run:659, ThreadPoolExecutor$Worker (org.apache.tomcat.util.threads)
run:61, TaskThread$WrappingRunnable (org.apache.tomcat.util.threads)
run:829, Thread (java.lang)

@Override
public void service(org.apache.coyote.Request req, org.apache.coyote.Response res) throws Exception {
    // ...
    postParseSuccess = postParseRequest(req, request, res, response);
    if (postParseSuccess) {
        // Calling the container
        connector.getService().getContainer().getPipeline().getFirst().invoke(request, response);
    }
}

private void internalMap(CharChunk host, CharChunk uri, String version, MappingData mappingData) throws IOException {
    if (uri.isNull()) {
        // Can't map context or wrapper without a uri
        return;
    }
    uri.setLimit(-1);

    // Context mapping
    ContextList contextList = mappedHost.contextList;
    MappedContext[] contexts = contextList.contexts;
    int pos = find(contexts, uri);
    if (pos == -1) {
        return;
    }
    // Wrapper mapping
    if (!contextVersion.isPaused()) {
        internalMapWrapper(contextVersion, uri, mappingData);
    }

}

http://host:port/context-root[/url-pattern]

private void internalMapWrapper(ContextVersion contextVersion, CharChunk path, MappingData mappingData) throws IOException {
    // Rule 1 -- Exact Match
    MappedWrapper[] exactWrappers = contextVersion.exactWrappers;
    internalMapExactWrapper(exactWrappers, path, mappingData);
    // Rule 2 -- Prefix Match
    MappedWrapper[] wildcardWrappers = contextVersion.wildcardWrappers;
    if (mappingData.wrapper == null) {
        internalMapWildcardWrapper(wildcardWrappers, contextVersion.nesting, path, mappingData);
    }
    // Rule 3 -- Extension Match
    MappedWrapper[] extensionWrappers = contextVersion.extensionWrappers;
    if (mappingData.wrapper == null && !checkJspWelcomeFiles) {
        internalMapExtensionWrapper(extensionWrappers, path, mappingData, true);
    }
    // Rule 4 -- Welcome resources processing for servlets
        // Rule 4a -- Welcome resources processing for exact macth
        // Rule 4b -- Welcome resources processing for prefix match
        // Rule 4c -- Welcome resources processing

    /*
    * welcome file processing - take 2 Now that we have looked for welcome files with a physical backing, now look
    * for an extension mapping listed but may not have a physical backing to it. This is for the case of index.jsf,
    * index.do, etc. A watered down version of rule 4
    */

    // Rule 7 -- Default servlet
}

protected boolean postParseRequest(){
    MessageBytes decodedURI = req.decodedURI();
    // Parse (and strip out) the path parameters
    parsePathParameters(req, request);
    // URI decoding
    // %xx decoding of the URL
    req.getURLDecoder().convert(decodedURI.getByteChunk(),
            connector.getEncodedSolidusHandlingInternal());
    // Normalization
    if (normalize(req.decodedURI())) {
        // Character decoding
        convertURI(decodedURI, request);
    }
}

doGet:17, FlagServlet (com.example.resin)
service:120, HttpServlet (javax.servlet.http)
service:97, HttpServlet (javax.servlet.http)
doFilter:109, ServletFilterChain (com.caucho.server.dispatch)
doFilter:21, AdminFilter (com.example.resin)
doFilter:127, HttpFilter (javax.servlet.http)
doFilter:89, FilterFilterChain (com.caucho.server.dispatch)
doFilter:156, WebAppFilterChain (com.caucho.server.webapp)
doFilter:95, AccessLogFilterChain (com.caucho.server.webapp)
service:304, ServletInvocation (com.caucho.server.dispatch)
handleRequest:840, HttpRequest (com.caucho.server.http)
dispatchRequest:1367, TcpSocketLink (com.caucho.network.listen)
handleRequest:1323, TcpSocketLink (com.caucho.network.listen)
handleRequestsImpl:1307, TcpSocketLink (com.caucho.network.listen)
handleRequests:1215, TcpSocketLink (com.caucho.network.listen)
handleAcceptTaskImpl:1011, TcpSocketLink (com.caucho.network.listen)
runThread:117, ConnectionTask (com.caucho.network.listen)
run:93, ConnectionTask (com.caucho.network.listen)
handleTasks:175, SocketLinkThreadLauncher (com.caucho.network.listen)
run:61, TcpSocketAcceptThread (com.caucho.network.listen)
runTasks:173, ResinThread2 (com.caucho.env.thread2)
run:118, ResinThread2 (com.caucho.env.thread2)

public boolean handleRequest() throws IOException {
  startRequest();
  if (! parseRequest()) {
      return false;
  }
  CharSequence host = getInvocationHost();
  Invocation invocation = getInvocation(host, _uri, _uriLength);

  startInvocation();
  invocation.service(requestFacade, getResponseFacade());
}

// skip 'http:'
if (ch != '/') {
}
// read URI
while (! isHttpWhitespace[ch]) {
  uriBuffer[uriLength++] = (byte) ch;
  if (readTail <= readOffset) {
    uriBuffer = _uri;
    uriLength = _uriLength;
}
  ch = readBuffer[readOffset++] & 0xff;
}

public void splitQueryAndUnescape(Invocation invocation,
                                  byte []rawURI, int uriLength)
  throws IOException
{
  for (int i = 0; i < uriLength; i++) {
    if (rawURI[i] == '?') {
      i++;

      // XXX: should be the host encoding?
      String queryString = byteToChar(rawURI, i, uriLength - i,
                                      "ISO-8859-1");
      invocation.setQueryString(queryString);

      uriLength = i - 1;
      break;
    }
  }

  String rawURIString = byteToChar(rawURI, 0, uriLength, "ISO-8859-1");
  invocation.setRawURI(rawURIString);

  String decodedURI = normalizeUriEscape(rawURI, 0, uriLength, _encoding);

  if (_sessionSuffix != null) {
    int p = decodedURI.indexOf(_sessionSuffix);

    if (p >= 0) {
      int suffixLength = _sessionSuffix.length();
      int tail = decodedURI.indexOf(';', p + suffixLength);
      String sessionId;

      if (tail > 0)
        sessionId = decodedURI.substring(p + suffixLength, tail);
      else
        sessionId = decodedURI.substring(p + suffixLength);

      decodedURI = decodedURI.substring(0, p);

      invocation.setSessionId(sessionId);

      p = rawURIString.indexOf(_sessionSuffix);
      if (p > 0) {
        rawURIString = rawURIString.substring(0, p);
        invocation.setRawURI(rawURIString);
      }
    }
  }
  else if (_sessionPrefix != null) {
    if (decodedURI.startsWith(_sessionPrefix)) {
      int prefixLength = _sessionPrefix.length();

      int tail = decodedURI.indexOf('/', prefixLength);
      String sessionId;

      if (tail > 0) {
        sessionId = decodedURI.substring(prefixLength, tail);
        decodedURI = decodedURI.substring(tail);
        invocation.setRawURI(rawURIString.substring(tail));
      }
      else {
        sessionId = decodedURI.substring(prefixLength);
        decodedURI = "/";
        invocation.setRawURI("/");
      }

      invocation.setSessionId(sessionId);
    }
  }

  String uri = normalizeUri(decodedURI);

  invocation.setURI(uri);
  invocation.setContextURI(uri);
}

private static int scanUriEscape(ByteToChar converter,
                                 byte []rawUri, int i, int len)
  throws IOException
{
  int ch1 = i < len ? (rawUri[i++] & 0xff) : -1;

  if (ch1 == 'u') {
    ch1 = i < len ? (rawUri[i++] & 0xff) : -1;
    int ch2 = i < len ? (rawUri[i++] & 0xff) : -1;
    int ch3 = i < len ? (rawUri[i++] & 0xff) : -1;
    int ch4 = i < len ? (rawUri[i++] & 0xff) : -1;

    converter.addChar((char) ((toHex(ch1) << 12) +
                              (toHex(ch2) << 8) + 
                              (toHex(ch3) << 4) + 
                              (toHex(ch4))));
  }
  else {
    int ch2 = i < len ? (rawUri[i++] & 0xff) : -1;

    int b = (toHex(ch1) << 4) + toHex(ch2);;

    converter.addByte(b);
  }

  return i;
}

public String normalizeUri(String uri, boolean isWindows)
  throws IOException
{
  CharBuffer cb = new CharBuffer();

  int len = uri.length();

  if (_maxURILength < len)
    throw new BadRequestException(L.l("The request contains an illegal URL because it is too long."));

  char ch;
  if (len == 0 || (ch = uri.charAt(0)) != '/' && ch != '\\')
    cb.append('/');

  for (int i = 0; i < len; i++) {
    ch = uri.charAt(i);

    if (ch == '/' || ch == '\\') {
    dots:
      while (i + 1 < len) {
        ch = uri.charAt(i + 1);

        if (ch == '/' || ch == '\\')
          i++;
        else if (ch != '.')
          break dots;
        else if (len <= i + 2
                 || (ch = uri.charAt(i + 2)) == '/' || ch == '\\') {
          i += 2;
        }
        else if (ch != '.')
          break dots;
        else if (len <= i + 3
                 || (ch = uri.charAt(i + 3)) == '/' || ch == '\\') {
          int j;

          for (j = cb.length() - 1; j >= 0; j--) {
            if ((ch = cb.charAt(j)) == '/' || ch == '\\')
              break;
          }
          if (j > 0)
            cb.setLength(j);
          else
            cb.setLength(0);
          i += 3;
        } else {
          throw new BadRequestException(L.l("The request contains an illegal URL."));
        }
      }

      while (isWindows && cb.getLength() > 0
             && ((ch = cb.getLastChar()) == '.' || ch == ' ')) {
        cb.setLength(cb.getLength() - 1);

        if (cb.getLength() > 0
            && (ch = cb.getLastChar()) == '/' || ch == '\\') {
          cb.setLength(cb.getLength() - 1);
          // server/003n
          continue;
        }
      }

      cb.append('/');
    }
    else if (ch == 0)
      throw new BadRequestException(L.l("The request contains an illegal URL."));
    else
      cb.append(ch);
  }

  while (isWindows && cb.getLength() > 0
         && ((ch = cb.getLastChar()) == '.' || ch == ' ')) {
    cb.setLength(cb.getLength() - 1);
  }

  return cb.toString();
}

mapServlet:234, ServletMapper (com.caucho.server.dispatch)
buildInvocation:4154, WebApp (com.caucho.server.webapp)
buildInvocation:798, WebAppContainer (com.caucho.server.webapp)
buildInvocation:753, Host (com.caucho.server.host)
buildInvocation:319, HostContainer (com.caucho.server.host)
buildInvocation:1064, ServletService (com.caucho.server.cluster)
buildInvocation:250, InvocationServer (com.caucho.server.dispatch)
buildInvocation:223, InvocationServer (com.caucho.server.dispatch)
buildInvocation:1610, AbstractHttpRequest (com.caucho.server.http)
getInvocation:1583, AbstractHttpRequest (com.caucho.server.http)
handleRequest:822, HttpRequest (com.caucho.server.http)

public FilterChain mapServlet(ServletInvocation invocation) throws ServletException {
    String servletName = null;
    String contextURI = invocation.getContextURI();
    if (_servletMap != null) {
        cleanUri = Invocation.stripPathParameters(contextURI);
        ServletMapping servletMap = _servletMap.map(cleanUri, vars);

        if (servletMap != null) {
            servletName = servletMap.getServletName();
        }
    }

    if (servletName == null) {
        InputStream is;
        is = _webApp.getResourceAsStream(contextURI);
        if (is != null) {
            is.close();
            servletName = _defaultServlet;
        }
    }
    MatchResult matchResult = null;
    if (matchResult == null && contextURI.endsWith("j_security_check")) {
      servletName = "j_security_check";
    }

    if (servletName == null) {
      servletName = _defaultServlet;
      vars.clear();
      if (matchResult != null)
        vars.add(matchResult.getContextUri());
      else
        vars.add(contextURI);
      addWelcomeFileDependency(invocation);
    }

    if (servletName == null) {
      log.fine(L.l("'{0}' has no default servlet defined", contextURI));
      return new ErrorFilterChain(404);
    }

    String servletPath = vars.get(0);
    invocation.setServletPath(servletPath);
    invocation.setPathInfo(/*...*/);

    if (servletName.equals("invoker"))
      servletName = handleInvoker(invocation);

    invocation.setServletName(servletName);
    ServletConfigImpl newConfig = _servletManager.getServlet(servletName);
    if (newConfig != null) config = newConfig;

    FilterChain chain = _servletManager.createServletChain(servletName, config, invocation);

    if (chain instanceof PageFilterChain) {
      PageFilterChain pageChain = (PageFilterChain) chain;
      chain = PrecompilePageFilterChain.create(invocation, pageChain);
    }

    return chain;
}

public InputStream getResourceAsStream(String uripath)
{
  Path rootDirectory = getRootDirectory();
  Path path = rootDirectory.lookupNative(getRealPath(uripath));

    if (path.canRead())
      return path.openRead();
    else {
      String resource = "META-INF/resources" + uripath;
      return getClassLoader().getResourceAsStream(resource);
    }
}

private void checkName(int i, String filename) {
    File f = new File(filename);
    if (f.exists())
    System.out.println(String.format("%06x pass: [%s]", i, filename));
}

public void fuzzFilename(String originFile) {
    File file = new File(originFile);
    if ( !file.exists() ) {
        return;
    }
    System.out.println(originFile + " exists.");
    Path filePath = Paths.get(originFile);
    String name = filePath.getFileName().toString();
    Path parentPath = filePath.getParent();
    String dirName = parentPath != null ? parentPath.toString() : "";
    System.out.println(String.format("dirName: %s, name: %s", dirName, name));

    for (int i = 0; i <= 0x10FFFF; i++) {
        // if (!Character.isValidCodePoint(i)) continue;
        String fuzzChar = new String(Character.toChars(i));
        checkName(i, originFile + fuzzChar);
        checkName(i, fuzzChar + originFile);
        checkName(i, dirName + fuzzChar + name);
        checkName(i, dirName + fuzzChar + File.separator + name);
        checkName(i, dirName + File.separator + fuzzChar + name);
    }
}

dirName: /tmp, name: 1.txt
00002f pass: [/tmp/1.txt/]
00002f pass: [//tmp/1.txt]
00002f pass: [/tmp//1.txt]

test\1.txt exists.
dirName: test, name: 1.txt
000020 pass: [test\1.txt ]
00002e pass: [test.\1.txt]
00002e pass: [test\1.txt.]
00002f pass: [test/1.txt]
00002f pass: [test/\1.txt]
00002f pass: [test\/1.txt]
00002f pass: [test\1.txt/]
00005c pass: [test\1.txt\]
00005c pass: [test\1.txt]
00005c pass: [test\\1.txt]

<文件名>:<数据流名>:<数据流类型>

  foo://example.com:8042/over/there?name=ferret#nose
  \_/   \______________/\_________/ \_________/ \__/
   |           |            |            |        |
scheme     authority       path        query   fragment
   |   _____________________|__
  / \ /                        \
  urn:example:animal:ferret:nose

path          = path-abempty    ; begins with "/" or is empty
              / path-absolute   ; begins with "/" but not "//"
              / path-noscheme   ; begins with a non-colon segment
              / path-rootless   ; begins with a segment
              / path-empty      ; zero characters

path-abempty  = *( "/" segment )
path-absolute = "/" [ segment-nz *( "/" segment ) ]
path-noscheme = segment-nz-nc *( "/" segment )
path-rootless = segment-nz *( "/" segment )
path-empty    = 0<pchar>

segment       = *pchar
segment-nz    = 1*pchar
segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
            ; non-zero-length segment without any colon ":"

pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"

@WebFilter(filterName = "AdminFilter", urlPatterns = "/*")
public class AdminFilter extends HttpFilter {
    @Override
    public void doFilter(HttpServletRequest req, HttpServletResponse res, FilterChain chain) throws IOException, ServletException {
        String uri = req.getRequestURI();
        if (uri.startsWith("/api/flag")) {
            res.setStatus(403);
            res.getWriter().write("YOU SHALL NOT PASS: " + uri);
        } else {
            chain.doFilter(req, res);
        }
    }
}

public String path(String path) {
   path = this.uriDecode(path);
   if (path != null && path.indexOf("\\") != -1) {
      path = StringUtil.replace(path, "\\", "/", false);
   }

   if (path != null && path.indexOf("..") != -1) {
      path = StringUtil.replace(path, "\\.{2,}", "");
   }

   if (path != null && path.indexOf("./") != -1) {
      path = StringUtil.replace(path, "./", "", false);
   }

   if (path != null && path.indexOf(";") != -1) {
      path = StringUtil.replace(path, ";.*?/", "/");
   }

   if (path != null && path.indexOf(";") != -1) {
      path = StringUtil.replace(path, ";.*", "");
   }

   if (path != null && path.indexOf("//") != -1) {
      path = StringUtil.replace(path, "/{2,}", "/");
   }

   if (path != null) {
      path = StringUtil.replace(path, "\\s", "");
   }

   if (!path.equals("") && !path.startsWith("/")) {
      path = "/" + path;
   }

   return path;
}
private String uriDecode(String path) {
  try {
      if (path != null && path.indexOf("%") != -1) {
        return URLDecoder.decode(path);
      }
  } catch (Exception var3) {
      this.writeLog("uri decode error:" + path, true);
      this.writeError(var3);
  }

  return path;
}

<listener>
    <listener-class>org.apache.shiro.web.env.EnvironmentLoaderListener</listener-class>
</listener>

<filter>
    <filter-name>ShiroFilter</filter-name>
    <filter-class>org.apache.shiro.web.servlet.ShiroFilter</filter-class>
</filter>

<filter-mapping>
    <filter-name>ShiroFilter</filter-name>
    <url-pattern>/*</url-pattern>
    <dispatcher>REQUEST</dispatcher>
    <dispatcher>FORWARD</dispatcher>
    <dispatcher>INCLUDE</dispatcher>
    <dispatcher>ERROR</dispatcher>
    <dispatcher>ASYNC</dispatcher>
</filter-mapping>

[main]
authc.loginUrl = /login.jsp
filterChainResolver.globalFilters = null

[urls]
/login/* = anon
/admin/* = authc
/api/flag = authc

[users]
evilpan=111111

getPathWithinApplication:114, WebUtils (org.apache.shiro.web.util)
getPathWithinApplication:105, PathMatchingFilter (org.apache.shiro.web.filter)
pathsMatch:124, PathMatchingFilter (org.apache.shiro.web.filter)
preHandle:195, PathMatchingFilter (org.apache.shiro.web.filter)
doFilterInternal:131, AdviceFilter (org.apache.shiro.web.servlet)
doFilter:154, OncePerRequestFilter (org.apache.shiro.web.servlet)
doFilter:66, ProxiedFilterChain (org.apache.shiro.web.servlet)
executeChain:458, AbstractShiroFilter (org.apache.shiro.web.servlet)
call:373, AbstractShiroFilter$1 (org.apache.shiro.web.servlet)
doCall:90, SubjectCallable (org.apache.shiro.subject.support)
call:83, SubjectCallable (org.apache.shiro.subject.support)
execute:387, DelegatingSubject (org.apache.shiro.subject.support)
doFilterInternal:370, AbstractShiroFilter (org.apache.shiro.web.servlet)
doFilter:154, OncePerRequestFilter (org.apache.shiro.web.servlet)
internalDoFilter:178, ApplicationFilterChain (org.apache.catalina.core)

public static String getPathWithinApplication(HttpServletRequest request) {
    return normalize(removeSemicolon(getServletPath(request) + getPathInfo(request)));
}

private static String removeSemicolon(String uri) {
    int semicolonIndex = uri.indexOf(';');
    return (semicolonIndex != -1 ? uri.substring(0, semicolonIndex) : uri);
}

// org.apache.shiro.web.filter.mgt.PathMatchingFilterChainResolver#getChain
final String requestURI = getPathWithinApplication(request);
final String requestURINoTrailingSlash = removeTrailingSlash(requestURI);
for (String pathPattern : filterChainManager.getChainNames()) {
  if (pathMatches(pathPattern, requestURI)) {
    return filterChainManager.proxy(originalChain, pathPattern);
  } else {
    // 针对 Spring Web 的处理
    pathPattern = removeTrailingSlash(pathPattern);
      if (pathMatches(pathPattern, requestURINoTrailingSlash)) {
        return filterChainManager.proxy(originalChain, pathPattern);
      }
  }
}