nginx+confd+etcd的配置自动管理

一、配置文件

1、etcd配置数据：

/apps/svr/etcd/bin/etcdctl set /service_sgin/upstream/service1/server1 192.168.1.1 
/apps/svr/etcd/bin/etcdctl set /service_sgin/subdomain/service1  "www.service1.com"

2、confd安装：

在github上下载：https://github.com/kelseyhightower/confd/releases 

在/etc/confd/下创建confd配置文件mkdir -p /etc/confd/{conf.d,templates}，具体配置如下：

[root@centos7-cloud ~]# cat /etc/confd/conf.d/nginx.conf.toml 
[template]
prefix = "/service_sgin/"   #此配置对应etcd的默认目录
src = "nginx.conf.tmpl"    #对应templates下的模板文件
dest = '/apps/conf/nginx/nginx.conf'   #对应nginx的配置文件
keys = [
  "/upstream",   #监控的键值目录
  "/subdomain",  #监控的键值目录
]
#check_cmd = "/apps/svr/nginx/sbin/nginx -t -c {{.src}}"   #检测到变化后的测试
#reload_cmd = "/apps/svr/nginx/sbin/nginx -s reload"       #配置文件没问题后的reload

[root@centos7-cloud ~]# 
[root@centos7-cloud ~]# cat /etc/confd/
conf.d/     confd.toml  templates/  
[root@centos7-cloud ~]# cat /etc/confd/confd.toml 
backend = "etcd"
confdir = "/etc/confd/"
interval = 5
#watch = true
log-level = "info"
nodes = [
    "http://centos7-cloud:2379",
]
noop = false
prefix = "/"
scheme = "http"

[root@centos7-cloud ~]# cat /etc/confd/templates/nginx.conf.tmpl 
user  apps apps;

worker_processes 24;
worker_cpu_affinity 000000000000000000000001;

error_log  /apps/logs/nginx/nginx_error.log  error;


#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65535;

events
{
  use epoll;
  worker_connections 65535;
}

http
{
  include       mime.types;
  #geoip_city /usr/local/nginx/conf/GeoLiteCity.dat;
  #include geo.conf;
  default_type  application/octet-stream;

  #charset  gb2312;


  sendfile on;
  tcp_nopush     on;

  client_body_timeout  5;
  client_header_timeout 5;
  keepalive_timeout  5;
  send_timeout       5;

  open_file_cache max=65535 inactive=20s;
  open_file_cache_valid 30s;
  open_file_cache_min_uses 1;

  tcp_nodelay on;

  fastcgi_connect_timeout 300;
  fastcgi_send_timeout 300;
  fastcgi_read_timeout 300;
  fastcgi_buffer_size 64k;
  fastcgi_buffers 4 64k;
  fastcgi_busy_buffers_size 128k;
  fastcgi_temp_file_write_size 128k;


 
  gzip on;
  gzip_min_length  1k;
  gzip_buffers     4 16k;
  gzip_http_version 1.0;
  gzip_comp_level 2;
  gzip_types       text/plain application/x-javascript text/css application/xml text/javascript application/json application/javascript;
  gzip_vary on;

  #limit_zone  crawler  $binary_remote_addr  10m;
  proxy_temp_path   /dev/shm/temp;
  proxy_cache_path  /dev/shm/cache levels=2:2:2   keys_zone=cache_go:200m inactive=5d max_size=7g;

  log_format log_access "$remote_addr" "\t$remote_user" "\t$time_local" "\t$request" "\t$request_time" "\t$upstream_response_time" "\t$status" "\t$body_bytes_sent";


############################################################################################################################################################## 

######################对应upstream的模板部分################
{{range $sub := ls "/subdomain"}}
    upstream {{base $sub}} {
{{$subdir := printf "/upstream/%s/*" $sub}}{{range getvs $subdir}}
        server {{.}}; {{end}}
        keepalive_timeout 65s;
    }
{{end}}

#####################对应server的模板部分####################
{{range gets "/subdomain/*"}}
    server {
        listen  80;
        server_name  {{.Value}};
        location / {
                proxy_pass              http://{{base .Key}};
                proxy_redirect  off;
                proxy_set_header  Host                   $host;
                proxy_set_header  X-Real-IP             $remote_addr;
                proxy_set_header  X-Forwarded-For  $proxy_add_x_forwarded_for;
          }
    }
{{end}}
}

3、nginx安装配置，这里不多说

4、执行confd，生成配置：

[root@centos7-cloud bin]#  ./confd -onetime -backend etcd -node http://127.0.0.1:2379
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Backend set to etcd
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Starting confd
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Backend nodes set to http://127.0.0.1:2379
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO /apps/conf/nginx/nginx.conf has md5sum 391c4e48a2c484599896dbac3e0633c5 should be 7601c4d759f61e963075c38f532f5b36
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:40:10Z centos7-cloud ./confd[21466]: INFO Target config /apps/conf/nginx/nginx.conf has been updated

[root@centos7-cloud bin]# ./confd -interval=60 -backend etcd -node http://127.0.0.1:2379 &
[1] 21501
[root@centos7-cloud bin]# 2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Backend set to etcd
2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Starting confd
2017-06-17T15:40:27Z centos7-cloud ./confd[21501]: INFO Backend nodes set to http://127.0.0.1:2379

使用onetime参数的，配置文件生成一次后，confd程序就退出了，下面的那句，会每隔60秒轮询一次。一旦后端etcd相应的值发生变化就会重新生成相应的配置文件。

二、测试：

1、通过etcdctl命令，往etcd里增加k/v：

[root@centos7-cloud ~]# /apps/svr/etcd/bin/etcdctl set /service_sgin/upstream/service1/server3 192.168.1.3
192.168.1.3
[root@centos7-cloud ~]# /apps/svr/etcd/bin/etcdctl set /service_sgin/subdomain/service3  "www.service3.com" 
www.service3.com

2、等待60s：

confd打印出来update更新的日志：

 2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO /apps/conf/nginx/nginx.conf has md5sum 7601c4d759f61e963075c38f532f5b36 should be 5f0e6e8a0bb2fb8163883318094c9fb7
2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:41:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf has been updated
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO /apps/conf/nginx/nginx.conf has md5sum 5f0e6e8a0bb2fb8163883318094c9fb7 should be 7da66fc7dd91dc65817744ea1fca23a8
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf out of sync
2017-06-17T15:42:27Z centos7-cloud ./confd[21501]: INFO Target config /apps/conf/nginx/nginx.conf has been updated

3、对比文件：

初始：

自动更新后：