openresty+redis实现 redis存储upstream

任务描述

1. upstream的server实例存储在redis中，为openresty设置一个location，来触发加载server的动作
2. 访问upstream时，轮询内部的server
3. 不包括健康检测，我认为既然你的server都存储在redis里面了，健康检测也应该独立于openresty，发现坏节点，重新加载
   

技术描述

ngx.balancer      ngx.shared.DICT    resty.redis 

部署环境描述

1. http://10.161.4.61:80/  后端实例
2. http://10.161.4.63:80/  后端实例
3. http://10.161.4.63:8000/  openresty
4. 10.161.4.63 6379   redis
   

步骤

部署redis

启动一个redis实例，没有特殊要求，能访问就行

设置openresty

修改配置文件

worker_processes  3;
error_log logs/error.log;
events {
    worker_connections  30;
}
http {
	lua_shared_dict  servers 10m;#全局变量，存储upstream，根据你的后端节点数量，设置大小
	lua_shared_dict  now_server 1m;#全局变量，用于轮询，记录现在是哪个节点服务
	lua_shared_dict  lens 1m;#全局变量，记录后端节点的数量
	upstream  tomcat17 {#这个upstream用于对比测试
    		server   10.161.4.63:80 weight=1;
    		server   10.161.4.61:80 weight=1;
    		keepalive 10;
        }
    upstream test{#动态加载的upstream
    	server   192.168.0.18:80 weight=1;#此处必须有一个server，否则会启动失败，可以胡乱写一个
    	balancer_by_lua_block {
    		local set_peer = require "set_peer"
			set_peer.run_peer()
     	}
     	keepalive 10;
    }
    #init_worker_by_lua_block {
	#		local set_peer = require "set_peer"
	#		set_peer.insert_server()   
	#}
	server{
		listen 8000;
		location ~ ^/api/([-_a-zA-Z0-9/]+) {
			access_by_lua_file work/lua/check_access.lua;
			content_by_lua_file work/lua/$1.lua;
		}
		location = /tomcat17 {#用于对比测试的location
		     proxy_pass http://tomcat17/;
		}
		location = /insert {#用于加载server的location
			content_by_lua_block{
				local set_peer = require "set_peer"
				set_peer.insert_server()
			}
		}
		location = /test_run {#用于测试的upstream
			proxy_pass http://test/;
		        
		}

		location = /status {
			stub_status;
		}
	}
}

lualib或者你自己的lua目录下创建，set_peer.lua 脚本（刚学lua没多久，写的有点水）

local balancer = require "ngx.balancer"
local now_server = ngx.shared.now_server;
local servers = ngx.shared.servers;
local server_len = ngx.shared.lens;
local redis = require "resty.redis"

local _M = { _VERSION = '1.0' }

local function con_redis()
	local red = redis:new()
	red:set_timeout(1000)
	local ok, err = red:connect("127.0.0.1", 6379)--注意此处
    if not ok then
        ngx.log(ngx.ERR,"failed to connect: ", err)
        return ngx.exit(500)
    else
    	return red
    end
end

local function set_peer(host,port)
	if not host and not port then
	    ngx.log(ngx.ERR, " host and port can not be nil")
        return ngx.exit(500)
	end

	local ok, err = balancer.set_current_peer(host, port)
    if not ok then
        ngx.log(ngx.ERR, "failed to set the current peer: ", err)
        return ngx.exit(500)
    end
end

local function split( str,reps)
    local resultStrList = {}
    string.gsub(str,'[^'..reps..']+',function ( w )
        table.insert(resultStrList,w)
    end)
    return resultStrList
end

local function test_server()
	 local value, err = servers:get(1)
	 if  not value then
	 	ngx.log(ngx.ERR, "no server found: ", err)
	 	ngx.status=ngx.HTTP_INTERNAL_SERVER_ERROR 
	 	return ngx.say("no server found")
	 else
	 	return true
	 end
end

local function fail_say( fail_str )
	-- body
	ngx.log(ngx.ERR, fail_str)
	ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR 
	return ngx.say(fail_str)
end

local function get_lens()
	-- body
	local len, flags = server_len:get("len")
	if not len then
		local fail_str = "failed to get len "..flags
		fail_say(fail_str)
	end
	return len
end

local function get_nums()
	local value = now_server:get("nums")
	if not value then
		local fail_str = 'reset  nums  to 1'
		ngx.log(ngx.ERR, fail_str)
		local ok, err = now_server:set("nums",1)
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return -1
		end
	end
	return value
end

local function round_server()
	-- body
	local value = get_nums()
	if not value then
		return
	end
	--ngx.log(ngx.ERR, '---->','ago',value)
	local len = get_lens()
	if not len then
		return
	end
	if value+1 >= len then
		local ok, err = now_server:set("nums",1)
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return 1
		end
	else
		local ok, err = now_server:set( "nums", 2+value )
		if not ok then
			fail_str = 'failed to set nums '..err
			fail_say(fail_str)
		else
			return 2+value
		end
	end 
end

local function  handle_redis(red)
	if not red then
		return 
	end
    local res, err = red:smembers("servers")
    if not res then
        ngx.log(ngx.ERR,"failed to smembers servers: ", err)
        return ngx.exit(500)
    else
    	return res
    end

end

function _M.insert_server()
	-- body
	local red = con_redis()
	if not red then
		return
	end
	local red_result = handle_redis(red)
	--ngx.log(ngx.ERR,'-->',red_result[1])
	if not red_result then
		red:close()
		return
	end
	local nums = 0
	for  k,v in pairs(red_result) do
		local resultStrList = split(v,':')
		--ngx.log(ngx.ERR,'-->', resultStrList[1], resultStrList[2])
		if  resultStrList[1] and  resultStrList[2] then
			local res, err = servers:set(2*k-1, resultStrList[1])
			if not res then
        		ngx.log(ngx.ERR,"failed to set servers first: ", err)
        		red:close()
        		return ngx.exit(500)
        	end
        	--ngx.log(ngx.ERR,'-->',2*k-1,'--->',resultStrList[1])
        	--ngx.log(ngx.ERR,'-->',2*k,'--->',resultStrList[2])
        	local res1, err1 = servers:set(2*k, resultStrList[2])
        	if not res1 then
        		ngx.log(ngx.ERR,"failed to set servers second: ", err1)
        		local res2, err2 = servers:rpop(k)
        		if not res2 then
        			ngx.log(ngx.ERR,"failed to rpop servers first: ", err2)
        		end
        		red:close()
        		return ngx.exit(500)
        	else
        		nums = nums+1
        		--ngx.log(ngx.ERR,'-->', nums)
        	end
		end
	end
	local res, err = server_len:set('len', nums*2)
	if not res then
        ngx.log(ngx.ERR,"failed to set servers first: ", err)
        return ngx.exit(500)
    else
    	ngx.say('ok,all is ', nums)
    	return true
    end
end

function _M.run_peer()
	local ok = test_server()
	if not ok then
		return		
	end
	local len = round_server()
	if not len then
		return		
	end
	ngx.log(ngx.ERR,'--->', len)--可以去掉，调试时使用
	local host ,err = servers:get(len)
	if not host then
		ngx.log(ngx.ERR,"failed to get server-host: ", err)
        return ngx.exit(500)
	end
	--ngx.log(ngx.ERR,"get server: ", host, '----', len+1)
	local port ,err = servers:get(len+1)
	if not port then
		ngx.log(ngx.ERR,"failed to get server-port: ", err)
        return ngx.exit(500)
	end
	ngx.log(ngx.ERR, "set peer ", host, ':', port) --可以去掉，调试时使用
	set_peer(host, port)
end

return _M

重启openresty

重启一下openresty，启动时监控一下errlog,确保启动成功

在redis内插入upstream的server

SADD servers '10.161.4.63:80' '10.161.4.61:80'

动态加载

curl -X GET http://10.161.4.63:8000/insert

正常的时候会得到 OK，all is 2 的句子

访问测试

连续两次执行，curl -X GET http://10.161.4.63:8000/test_run，查看结果是不是轮询

简单的压力测试

简单测试了一下，觉得性能差不多，仅供参考呀

----------------------------------------------------------------------------------------------------------------------------------

就写到这了，如果有问题，欢迎来qq群 630300475 一起聊聊