openresty+redis实现 redis存储upstream

冷不防 2022-05-20 01:29 283阅读 0赞

##### 任务描述 #####

1.  upstream的server实例存储在redis中，为openresty设置一个location，来触发加载server的动作
2.  访问upstream时，轮询内部的server
3.  不包括健康检测，我认为既然你的server都存储在redis里面了，健康检测也应该独立于openresty，发现坏节点，重新加载

##### 技术描述 #####

ngx.balancer      ngx.shared.DICT    resty.redis

##### 部署环境描述    #####

1.  http://10.161.4.61:80/  后端实例
2.  http://10.161.4.63:80/  后端实例
3.  http://10.161.4.63:8000/  openresty
4.  10.161.4.63 6379   redis

##### 步骤    #####

###### 部署redis ######

启动一个redis实例，没有特殊要求，能访问就行

###### 设置openresty    ######

修改配置文件

worker_processes  3;
    error_log logs/error.log;
    events {
        worker_connections  30;
    }
    http {
    	lua_shared_dict  servers 10m;#全局变量，存储upstream，根据你的后端节点数量，设置大小
    	lua_shared_dict  now_server 1m;#全局变量，用于轮询，记录现在是哪个节点服务
    	lua_shared_dict  lens 1m;#全局变量，记录后端节点的数量
    	upstream  tomcat17 {
       #这个upstream用于对比测试
        		server   10.161.4.63:80 weight=1;
        		server   10.161.4.61:80 weight=1;
        		keepalive 10;
            }
        upstream test{
       #动态加载的upstream
        	server   192.168.0.18:80 weight=1;#此处必须有一个server，否则会启动失败，可以胡乱写一个
        	balancer_by_lua_block {
        		local set_peer = require "set_peer"
    			set_peer.run_peer()
         	}
         	keepalive 10;
        }
        #init_worker_by_lua_block {
    	#		local set_peer = require "set_peer"
    	#		set_peer.insert_server()   
    	#}
    	server{
    		listen 8000;
    		location ~ ^/api/([-_a-zA-Z0-9/]+) {
    			access_by_lua_file work/lua/check_access.lua;
    			content_by_lua_file work/lua/$1.lua;
    		}
    		location = /tomcat17 {
       #用于对比测试的location
    		     proxy_pass http://tomcat17/;
    		}
    		location = /insert {
       #用于加载server的location
    			content_by_lua_block{
    				local set_peer = require "set_peer"
    				set_peer.insert_server()
    			}
    		}
    		location = /test_run {
       #用于测试的upstream
    			proxy_pass http://test/;
    		        
    		}
    
    		location = /status {
    			stub_status;
    		}
    	}
    }

###### lualib或者你自己的lua目录下创建，set\_peer.lua 脚本（刚学lua没多久，写的有点水） ######

local balancer = require "ngx.balancer"
    local now_server = ngx.shared.now_server;
    local servers = ngx.shared.servers;
    local server_len = ngx.shared.lens;
    local redis = require "resty.redis"
    
    local _M = { _VERSION = '1.0' }
    
    local function con_redis()
    	local red = redis:new()
    	red:set_timeout(1000)
    	local ok, err = red:connect("127.0.0.1", 6379)--注意此处
        if not ok then
            ngx.log(ngx.ERR,"failed to connect: ", err)
            return ngx.exit(500)
        else
        	return red
        end
    end
    
    local function set_peer(host,port)
    	if not host and not port then
    	    ngx.log(ngx.ERR, " host and port can not be nil")
            return ngx.exit(500)
    	end
    
    	local ok, err = balancer.set_current_peer(host, port)
        if not ok then
            ngx.log(ngx.ERR, "failed to set the current peer: ", err)
            return ngx.exit(500)
        end
    end
    
    local function split( str,reps)
        local resultStrList = {}
        string.gsub(str,'[^'..reps..']+',function ( w )
            table.insert(resultStrList,w)
        end)
        return resultStrList
    end
    
    local function test_server()
    	 local value, err = servers:get(1)
    	 if  not value then
    	 	ngx.log(ngx.ERR, "no server found: ", err)
    	 	ngx.status=ngx.HTTP_INTERNAL_SERVER_ERROR 
    	 	return ngx.say("no server found")
    	 else
    	 	return true
    	 end
    end
    
    local function fail_say( fail_str )
    	-- body
    	ngx.log(ngx.ERR, fail_str)
    	ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR 
    	return ngx.say(fail_str)
    end
    
    local function get_lens()
    	-- body
    	local len, flags = server_len:get("len")
    	if not len then
    		local fail_str = "failed to get len "..flags
    		fail_say(fail_str)
    	end
    	return len
    end
    
    local function get_nums()
    	local value = now_server:get("nums")
    	if not value then
    		local fail_str = 'reset  nums  to 1'
    		ngx.log(ngx.ERR, fail_str)
    		local ok, err = now_server:set("nums",1)
    		if not ok then
    			fail_str = 'failed to set nums '..err
    			fail_say(fail_str)
    		else
    			return -1
    		end
    	end
    	return value
    end
    
    local function round_server()
    	-- body
    	local value = get_nums()
    	if not value then
    		return
    	end
    	--ngx.log(ngx.ERR, '---->','ago',value)
    	local len = get_lens()
    	if not len then
    		return
    	end
    	if value+1 >= len then
    		local ok, err = now_server:set("nums",1)
    		if not ok then
    			fail_str = 'failed to set nums '..err
    			fail_say(fail_str)
    		else
    			return 1
    		end
    	else
    		local ok, err = now_server:set( "nums", 2+value )
    		if not ok then
    			fail_str = 'failed to set nums '..err
    			fail_say(fail_str)
    		else
    			return 2+value
    		end
    	end 
    end
    
    local function  handle_redis(red)
    	if not red then
    		return 
    	end
        local res, err = red:smembers("servers")
        if not res then
            ngx.log(ngx.ERR,"failed to smembers servers: ", err)
            return ngx.exit(500)
        else
        	return res
        end
    
    end
    
    function _M.insert_server()
    	-- body
    	local red = con_redis()
    	if not red then
    		return
    	end
    	local red_result = handle_redis(red)
    	--ngx.log(ngx.ERR,'-->',red_result[1])
    	if not red_result then
    		red:close()
    		return
    	end
    	local nums = 0
    	for  k,v in pairs(red_result) do
    		local resultStrList = split(v,':')
    		--ngx.log(ngx.ERR,'-->', resultStrList[1], resultStrList[2])
    		if  resultStrList[1] and  resultStrList[2] then
    			local res, err = servers:set(2*k-1, resultStrList[1])
    			if not res then
            		ngx.log(ngx.ERR,"failed to set servers first: ", err)
            		red:close()
            		return ngx.exit(500)
            	end
            	--ngx.log(ngx.ERR,'-->',2*k-1,'--->',resultStrList[1])
            	--ngx.log(ngx.ERR,'-->',2*k,'--->',resultStrList[2])
            	local res1, err1 = servers:set(2*k, resultStrList[2])
            	if not res1 then
            		ngx.log(ngx.ERR,"failed to set servers second: ", err1)
            		local res2, err2 = servers:rpop(k)
            		if not res2 then
            			ngx.log(ngx.ERR,"failed to rpop servers first: ", err2)
            		end
            		red:close()
            		return ngx.exit(500)
            	else
            		nums = nums+1
            		--ngx.log(ngx.ERR,'-->', nums)
            	end
    		end
    	end
    	local res, err = server_len:set('len', nums*2)
    	if not res then
            ngx.log(ngx.ERR,"failed to set servers first: ", err)
            return ngx.exit(500)
        else
        	ngx.say('ok,all is ', nums)
        	return true
        end
    end
    
    function _M.run_peer()
    	local ok = test_server()
    	if not ok then
    		return		
    	end
    	local len = round_server()
    	if not len then
    		return		
    	end
    	ngx.log(ngx.ERR,'--->', len)--可以去掉，调试时使用
    	local host ,err = servers:get(len)
    	if not host then
    		ngx.log(ngx.ERR,"failed to get server-host: ", err)
            return ngx.exit(500)
    	end
    	--ngx.log(ngx.ERR,"get server: ", host, '----', len+1)
    	local port ,err = servers:get(len+1)
    	if not port then
    		ngx.log(ngx.ERR,"failed to get server-port: ", err)
            return ngx.exit(500)
    	end
    	ngx.log(ngx.ERR, "set peer ", host, ':', port) --可以去掉，调试时使用
    	set_peer(host, port)
    end
    
    return _M

###### 重启openresty ######

重启一下openresty，启动时监控一下errlog,确保启动成功

###### 在redis内插入upstream的server    ######

SADD servers '10.161.4.63:80' '10.161.4.61:80'

###### 动态加载 ######

curl -X GET http://10.161.4.63:8000/insert

正常的时候会得到 **OK，all is 2** 的句子

###### 访问测试    ######

连续两次执行，curl -X GET http://10.161.4.63:8000/test\_run，查看结果是不是轮询

![70][]

简单的压力测试

![70 1][]

![70 2][]

简单测试了一下，觉得性能差不多，仅供参考呀

\----------------------------------------------------------------------------------------------------------------------------------

就写到这了，如果有问题，欢迎来qq群 630300475 一起聊聊

[70]: /images/20220520/78d4b55aba394bd2ac97819ab5519697.png
[70 1]: /images/20220520/504539d7c5eb4ff0a58bc12fc9658c7e.png
[70 2]: /images/20220520/74a0aaeea1e84a1bae2d2faaae9818a2.png