使用 etcd + confd 实现 nginx 自动代理后端服务

整体拓扑

1、10.3.20.142 confd 相关配置

# cat /etc/confd/conf.d/nginx.toml
[template]
src = "nginx.conf.tmpl"
dest = "/etc/nginx/stream.d/tcp.conf"
keys = [
    "/app/py-app/",
]
check_cmd = "/usr/sbin/nginx -t"
reload_cmd = "/usr/sbin/nginx -s reload"
 
 
# cat /etc/confd/templates/nginx.conf.tmpl
upstream py-app {
    {{range getvs "/app/py-app/upstream/*"}}
        server {{.}};
    {{end}}
}
 
server {
    listen 8088;
    proxy_pass py-app;
}

2、10.3.20.142 nginx tcp 代理相关配置

# egrep -v "^$|^#" /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
http {
// 省略部分...
}
stream {
    log_format proxy '$remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
  
    access_log  /var/log/nginx/proxy.log proxy ;
    include /etc/nginx/stream.d/*.conf;
}

3、10.3.20.142 启动 confd，使用 etcd 作为后端存储

# nohup confd-0.16.0-linux-amd64 -interval=10 -backend etcd  -node http://10.3.20.102:2379 > /tmp/confd.log 2>&1 &
 
 
// 后端服务还没启动，所以还没有通过模板生成 nginx tcp.conf 配置文件
# ll /etc/nginx/stream.d/tcp.conf
ls: cannot access /etc/nginx/stream.d/tcp.conf: No such file or directory

4、启动后端服务

// 启动 10.3.20.141 上的服务
# cat python-app-8088.py
from flask import Flask
import requests
import atexit
 
app = Flask(__name__)
 
etcd_url = "http://10.3.20.102:2379"
etcd_key = "/v2/keys/app/py-app/upstream/app1"
etcd_value = "10.3.20.141:8088"
 
 
def set_etcd_key():
    r = requests.put(etcd_url + etcd_key, data = {"value":etcd_value})
    return r.text
 
 
def del_etcd_key():
    r = requests.delete(etcd_url + etcd_key)
    return r.text
 
 
@app.route('/')
def hello_world():
    return 'app 10.3.20.141:8088'
 
 
if __name__ == '__main__':
    set_etcd_key()
    app.run(host="10.3.20.141", port="8088")
    atexit.register(del_etcd_key)
 
# python python-app-8088.py
 
// 启动 10.3.20.143 上的服务
# cat python-app-8088.py
from flask import Flask
import requests
import atexit
 
app = Flask(__name__)
 
etcd_url = "http://10.3.20.102:2379"
etcd_key = "/v2/keys/app/py-app/upstream/app2"
etcd_value = "10.3.20.143:8088"
 
 
def set_etcd_key():
    r = requests.put(etcd_url + etcd_key, data = {"value":etcd_value})
    return r.text
 
 
def del_etcd_key():
    r = requests.delete(etcd_url + etcd_key)
    return r.text
 
 
@app.route('/')
def hello_world():
    return 'app 10.3.20.143:8088'
 
 
if __name__ == '__main__':
    set_etcd_key()
    app.run(host="10.3.20.143", port="8088")
    atexit.register(del_etcd_key)
 
# python python-app-8088.py

5、测试 nginx 代理情况

// 10.3.20.142 confd 已经自动读取 etcd 的服务注册信息，并通过 /etc/confd/templates/nginx.conf.tmpl 自动生成 /etc/nginx/stream.d/tcp.conf，并 reload nginx 服务
# cat /etc/nginx/stream.d/tcp.conf
upstream py-app {
     
        server 10.3.20.141:8088;
     
        server 10.3.20.143:8088;
     
}
 
server {
    listen 8088;
    proxy_pass py-app;
}
 
 
# curl 10.3.20.142:8088
app 10.3.20.141:8088
# curl 10.3.20.142:8088
app 10.3.20.143:8088

参考：
etcd api：https://coreos.com/etcd/docs/latest/v2/api.html
atexit：https://docs.python.org/2.7/library/atexit.html
confd templates：https://github.com/kelseyhightower/confd/blob/master/docs/templates.md