OpenResty实战系列 | 异步非阻塞HTTP客户端库 Lua-Resty-Http

Tinywan

发布于 2024-07-30 16:09:29

1500

发布于 2024-07-30 16:09:29

文章被收录于专栏：开源技术小栈

概述

Lua-resty-http 是一个基于 OpenResty 的 Lua 库，是 OpenResty 项目中一个非常有用的模块，用于从 Nginx 服务中发起 HTTP 请求。OpenResty 是一个基于 Nginx 与 LuaJIT 的全功能 Web 平台，它集成了大量精心设计的 Nginx 模块，以及大量的 Lua 库。

lua-resty-http 库允许你在 OpenResty 的 Lua 环境中轻松地发送 HTTP 请求，它提供了一个简单易用的 API 来处理 HTTP 请求和响应。这使得在 Nginx 配置文件中编写 Lua 脚本来处理 HTTP 请求和响应成为可能，从而可以构建高性能的 Web 应用和服务。如果你正在使用 OpenResty 并需要在 Nginx 配置中发起 HTTP 请求，lua-resty-http 是一个非常合适的选择。

项目地址：https://github.com/ledgetech/lua-resty-http

特性

异步非阻塞：该库利用 nginx 的事件循环模型，让 HTTP 请求在后台执行，不会阻塞主线程，提高了整体性能。
连接池管理：它支持连接池的创建与管理，可以有效地复用 TCP 连接，减少握手延迟，提高服务响应速度。
丰富的 API 设计：Lua-Resty-HTTP 提供了一套完整的 API，包括设置超时、指定代理、添加请求头、处理重定向、自定义认证等，使得开发过程更为便捷。
SSL/TLS 支持：内置 SSL/TLS 功能，支持 HTTPS 请求，且允许自定义证书和密钥。
错误处理：提供了详细的错误信息，便于调试和故障排除。

应用场景

数据获取：从 RESTful API 获取 JSON 或其他格式的数据。
API 调用：在你的 OpenResty 应用中调用外部 Web 服务。
自动化测试：在 Lua 测试脚本中模拟 HTTP 请求，验证服务行为。
日志报告：向远程服务器发送日志或统计信息。
缓存刷新：根据 HTTP 响应自动更新本地缓存。

使用

安装

这里通过OPM工具包安装，更多请查看上一篇

opm get ledgetech/lua-resty-http

基础使用

使用 Lua-resty-http 发送 HTTP 请求的一个基本示例

local httpc = require("resty.http").new()

-- Single-shot requests use the `request_uri` interface.
local res, err = httpc:request_uri("http://example.com/helloworld", {
    method = "POST",
    body = "a=1&b=2",
    headers = {
        ["Content-Type"] = "application/x-www-form-urlencoded",
    },
})
if not res then
    ngx.log(ngx.ERR, "request failed: ", err)
    return
end

-- At this point, the entire request / response is complete and the connection
-- will be closed or back on the connection pool.

-- The `res` table contains the expeected `status`, `headers` and `body` fields.
local status = res.status
local length = res.headers["Content-Length"]
local body   = res.body

进阶使用

openresty.tinywan.com.conf配置文件

server {
    listen 80;
    server_name openresty.tinywan.com;

    location /lua_http_test {  
        default_type "text/html";
        lua_code_cache off; 
        content_by_lua_file conf/lua/lua_http_test.lua;
    }
}

lua_http_test.lua 脚本

local httpc = require("resty.http").new()

-- Single-shot requests use the `request_uri` interface.
local res, err = httpc:request_uri("https://www.workerman.net/u/Tinywan", {
    method = "GET",
    body = "name=Tinywan&age=24",
    headers = {
        ["Content-Type"] = "application/x-www-form-urlencoded",
    },
    ssl_verify = false,
})
if not res then
    ngx.log(ngx.ERR, "request failed: ", err)
    return
end

local status = res.status
local length = res.headers["Content-Length"]
local body   = res.body
ngx.say(res.body)

通过curl脚本测试请求打印结果

$ curl -i http://openresty.tinywan.com/lua_http_test
HTTP/1.1 200 OK
Server: openresty/1.17.8.2
Date: Wed, 17 Jul 2024 09:42:10 GMT
Content-Type: text/html; charset=utf-8
Transfer-Encoding: chunked
Connection: keep-alive
Vary: Accept-Encoding

<!doctype html>
<html lang="zh-cn">
<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

    <link rel="shortcut icon" href="/favicon.ico" />
    <link href="https://cdn.workerman.net/css/bootstrap.min.css?v=20211126" rel="stylesheet">
    <link href="https://cdn.workerman.net/css/main.css?v=20240705" rel="stylesheet">
    <script src="https://cdn.workerman.net/js/jquery.min.js"></script>
    <script src="https://cdn.workerman.net/js/bootstrap.min.js?v=20211126"></script>
    <script src="https://cdn.workerman.net/js/functions.js?v=20220507"></script>
    <script type="text/javascript" charset="UTF-8" src="https://cdn.wwads.cn/js/makemoney.js" async></script>

    <title>Tinywan的主页-分享-workerman社区</title>

</head>
...
</body>
</html>

项目应用

--[[--------------------------------------------------------      
* |  Copyright (C) Shaobo Wan (Tinywan)
* |  Origin: 开源技术小栈
* |-------------------------------------------------------- 
--]]

local helper = require "vendor.helper"
local redis = require "resty.redis"
local resty_lock = require "resty.lock"
local http = require "resty.http"
local log = ngx.log
local ERR = ngx.ERR
local live_ngx_cache = ngx.shared.live_ngx_cache
-- 非error 日志开关 1:开启，0:关闭
local log_switch = 1

local redis_host = "127.0.0.1"
local redis_port = 6379
local redis_auth = "123456"
local redis_timeout = 1000

-- set ngx.cache
local function set_cache(key, value, exptime)
    if not exptime then
        exptime = 0
    end
    local succ, err, forcible = live_ngx_cache:set(key, value, exptime)
    return succ
end

-- close redis
local function close_redis(red)
    if not red then
        return
    end
    --释放连接(连接池实现)
    local pool_max_idle_time = 10000 --毫秒
    local pool_size = 100 --连接池大小
    local ok, err = red:set_keepalive(pool_max_idle_time, pool_size)

    if not ok then
        log(ERR, "set redis keepalive error : ", err)
    end
end

-- read redis
local function read_redis(_host, _port, _auth, keys)
    local red = redis:new()
    red:set_timeout(redis_timeout)
    local ok, err = red:connect(_host, _port)
    if not ok then
        log(ERR, "connect to redis error : ", err)
    end

    -- 请注意这里 auth 的调用过程
    local count
    count, err = red:get_reused_times()
    if 0 == count then
        ok, err = red:auth(_auth)
        if not ok then
            log(ERR, "failed to auth: ", err)
            return close_redis(red)
        end
    elseif err then
        log(ERR, "failed to get reused times: ", err)
        return close_redis(red)
    end

    local resp = nil
    if #keys == 1 then
        resp, err = red:get(keys[1])
    else
        resp, err = red:mget(keys)
    end

    if not resp then
        log(ERR, keys[1] .. " get redis content error : ", err)
        return close_redis(red)
    end

    if resp == ngx.null then
        resp = nil
    end
    close_redis(red)
    if log_switch == 1 then
        log(ERR, "[2] [read_redis] content from redis.cache  id = " .. keys[1]) -- tag data origin
    end
    return resp
end

-- write redis
local function write_redis(_host, _port, _auth, keys, values)
    local red = redis:new()
    red:set_timeout(redis_timeout)
    local ok, err = red:connect(_host, _port)
    if not ok then
        log(ERR, "connect to redis error : ", err)
    end

    local count
    count, err = red:get_reused_times()
    if 0 == count then
        ok, err = red:auth(_auth)
        if not ok then
            log(ERR, "failed to auth: ", err)
            return close_redis(red)
        end
    elseif err then
        log(ERR, "failed to get reused times: ", err)
        return close_redis(red)
    end

    -- set data
    local resp = nil
    if #keys == 1 then
        resp, err = red:set(keys[1], values)
    else
        resp, err = red:mset(keys, values)
    end
    if not resp then
        log(ERR, "set redis live error : ", err)
        close_redis(red)
    end
    close_redis(red)
    return resp
end

-- get ngx.cache
--[1]即使发生其他一些不相关的错误，您也需要尽快解除锁定。
--[2]在释放锁之前，您需要从后端获得的结果更新缓存，以便其他已经等待锁定的线程在获得锁定后才能获得缓存值。
--[3]当后端根本没有返回任何值时，我们应该通过将一些存根值插入缓存来仔细处理。
local function read_cache(key)
    local ngx_resp = nil
    -- 获取共享内存上key对应的值。如果key不存在，或者key已经过期，将会返回nil；如果出现错误，那么将会返回nil以及错误信息。
    -- step 1
    local val, err = live_ngx_cache:get(key)
    if val then
        if log_switch == 1 then
            log(ERR, " [1] [read_ngx_cache]  content from ngx.cache id = " .. key) -- tag data origin
        end
        return val
    end

    if err then
        log(ERR, "failed to get key from shm: ", err)
    end

    -- cache miss!
    -- step 2:
    local lock, err = resty_lock:new("cache_lock") -- new resty.lock
    if not lock then
        log(ERR, "failed to create lock [cache_lock] : ", err)
        return
    end

    local elapsed, err = lock:lock(key) -- 锁
    if not elapsed then
        log(ERR, "failed to acquire the lock", err)
        return
    end
    -- lock successfully acquired!

    -- step 3:
    -- someone might have already put the value into the cache ,so we check it here again:
    val, err = live_ngx_cache:get(key)
    if val then
        local ok, err = lock:unlock()
        if not ok then
            log(ERR, "failed to unlock [111] : ", err)
        end
        return val
    end

    --- step 4:
    local val = read_redis(redis_host, redis_port, redis_auth, { key })
    if not val then
        local ok, err = lock:unlock()
        if not ok then
            log(ERR, "failed to unlock [222] : ", err)
        end
        -- FIXME: we should handle the backend miss more carefully
        -- here, like inserting a stub value into the cache.
        log(ERR, "[4] ngx.cache find redis cache no value found : ", err)
        return ngx_resp
    end

    -- [lock] update the shm cache with the newly fetched value
    local ok, err = live_ngx_cache:set(key, val, 1)
    if not ok then
        local ok, err = lock:unlock()
        if not ok then
            log(ERR, "failed to unlock [333] : ", err)
        end
        log(ERR, "failed to update live_ngx_cache:  ", err)
    end

    local ok, err = lock:unlock()
    if not ok then
        log(ERR, "failed to unlock [444] : ", err)
    end

    return val
end

-------------- read_http 大并发采用 resty.http ，对于：ngx.location.capture 慎用
local function read_http(id)
    local httpc = http.new()
    local resp, err = httpc:request_uri("https://live.tinywan.com", {
        method = "GET",
        path = "/api/live/" .. id,
        headers = {
            ["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36"
        }
    })

    if not resp then
        log(ERR, "resty.http API request error :", err)
        return
    end
    httpc:close()

    -- 判断状态码
    if resp.status ~= ngx.HTTP_OK then
        log(ERR, "request error, status :", resp.status)
        return
    end

    if resp.status == ngx.HTTP_FORBIDDEN then
        log(ERR, "request error, status :", resp.status)
        return
    end

    -- backend not data 判断api返回的状态码
    local status_code = helper.cjson_decode(resp.body)['code']
    if tonumber(status_code) == 200 then
        -- 正常数据缓存到 Redis 数据缓存
        local live_info_key = "LIVE_TABLE:" .. id
        local live_value = helper.cjson_decode(resp.body)['data'] -- 解析的Lua自己的然后存储到Redis 数据库中去(这里最好使用lua的json格式去写入)
        local live_live_str = write_redis(redis_host, redis_port, redis_auth, { live_info_key }, helper.cjson_encode(live_value))
        if not live_live_str then
            log(ERR, "redis set info error: ")
        end
        if log_switch == 1 then
            log(ERR, "[3] [read_http] content from backend API id : " .. id) -- tag data origin
        end
        return helper.cjson_encode(live_value)
    else
        -- 后端没有数据直接返回 nil
        log(ERR, " [read_http] backend API is not content return error_msg : " .. helper.cjson_decode(resp.body)['msg']) -- tag data origin
        return
    end
end

-- 业务逻辑处理
local function read_content(id)
    local cache_content = nil
    local live_key = "LIVE:" .. id
    local content = read_cache(live_key)

    if not content then
        log(ERR, "[5] redis not found content, back to backend API , id : ", id)
        content = read_http(id)
    end
    if not content then
        log(ERR, "backend API not found content, id : ", id)
        return cache_content
    end
    if tostring(content) == "false" then
        log(ERR, "backend API content is false ", id)
        return cache_content
    end
    return content
end

local _M = {
    read_content = read_content
}

return _M