在上一篇文章中，我们了解了saltstack搭建Web集群的基础环境以及haproxy的部署，这里我们将想继续了解一下其他的web服务如何通过saltstack部署起来。
参考资料：https://github.com/unixhot/saltbook-code
SaltStack部署keepalived
编写安装配置sls文件
在keepalived目标编写install.sls:
[root@node1 /srv/salt/prod/modules/keepalived]# cat install.sls
{% set keepalived_tar = 'keepalived-1.2.17.tar.gz' %} #利用pillar自定义变量和路径
{% set keepalived_source = 'salt://modules/keepalived/files/keepalived-1.2.17.tar.gz' %}
keepalived-install:
file.managed:
- name: /usr/local/src/{{ keepalived_tar }} #通过pillar变量定义，可以方便修改软件版本
- source: {{ keepalived_source }}
- mode: 755
- user: root
- group: root
cmd.run:
- name: cd /usr/local/src && tar zxf {{ keepalived_tar }} && \
cd keepalived-1.2.17 && ./configure --prefix=/usr/local/keepalived \
--disable-fwmark && make && make install
- unless: test -d /usr/local/keepalived #存在此目录则不执行name中的命令
- require:
- file: keepalived-install
/etc/sysconfig/keepalived:
file.managed:
- source: salt://modules/keepalived/files/keepalived.sysconfig
- mode: 644
- user: root
- group: root
/etc/init.d/keepalived:
file.managed:
- source: salt://modules/keepalived/files/keepalived.init
- mode: 755
- user: root
- group: root
keepalived-init:
cmd.run:
- name: chkconfig --add keepalived
- unless: chkconfig --list | grep keepalived
- require:
- file: /etc/init.d/keepalived
/etc/keepalived:
file.directory: # 目录管理，如果使用这个模块，指定的ID为一个目录
- user: root
- group: root
拷贝安装包和文件
将需要的安装包和配置文件拷贝到files目录：
[root@node1 /srv/salt/prod/modules/keepalived/files]# ll total 372 -rw-r--r--. 1 root root 368827 Nov 15 16:42 keepalived-1.2.17.tar.gz -rw-r--r--. 1 root root 1380 Nov 15 16:46 keepalived.init #启动脚本 -rw-r--r--. 1 root root 668 Nov 15 18:43 keepalived.sysconfig # sysconfig目录下的配置文件
[root@node1 /srv/salt/prod/modules/keepalived/files]# cat keepalived.sysconfig # Options for keepalived. See `keepalived --help' output and keepalived(8) and # keepalived.conf(5) man pages for a list of all options. Here are the most # common ones : # # --vrrp -P Only run with VRRP subsystem. # --check -C Only run with Health-checker subsystem. # --dont-release-vrrp -V Dont remove VRRP VIPs & VROUTEs on daemon stop. # --dont-release-ipvs -I Dont remove IPVS topology on daemon stop. # --dump-conf -d Dump the configuration data. # --log-detail -D Detailed log messages. # --log-facility -S 0-7 Set local syslog facility (default=LOG_DAEMON) # KEEPALIVED_OPTIONS="-D"
定义业务参数
对keepalived的配置文件使用pillar来定义修改的参数，其中参数引用了jinja模板：
[root@node1 /srv/salt/prod/cluster/files]# cat haproxy-outside-keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id `ROUTEID` #引入参数，在jinja中配置
}
vrrp_instance haproxy_ha {
state `STATEID`
interface eth0
virtual_router_id 36
priority `PRIORITYID`
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
172.16.10.62
}
}
定义这些参数的sls文件为：
[root@node1 /srv/salt/prod/cluster]# cat haproxy-outside-keepalived.sls
include:
- modules.keepalived.install # 执行keepalived的install文件
keepalived-server:
file.managed:
- name: /etc/keepalived/keepalived.conf
- source: salt://cluster/files/haproxy-outside-keepalived.conf
- mode: 644
- user: root
- group: root
- template: jinja
{% if grains['fqdn'] == 'node1' %} # 定义node1 为master
- ROUTEID: haproxy_ha
- STATEID: MASTER
- PRIORITYID: 150 # 优先级为150
{% elif grains['fqdn'] == 'node2' %}
- ROUTEID: haproxy_ha
- STATEID: BACKUP # 定义node2为backup
- PRIORITYID: 100 # 优先级100
{% endif %}
service.running:
- name: keepalived
- enable: True
- watch:
- file: keepalived-server
修改top file,加载对应的执行文件：
[root@node1 /srv/salt/base]# cat top.sls base: '*': - init.init prod: 'node*': - cluster.haproxy-outside - cluster.haproxy-outside-keepalived
执行salt命令：
salt '*' state.highstate test=True salt '*' state.highstate
提示：在配置中要一层层指定运行和调用的文件，对于业务上需要变动修改的文件，单独存放在/srv/salt/prod/cluster/files目录中，在它的上一级目录(也就是cluster目录)中存放sls文件,修改参数模板（jinja).
这里针对keepalived，使用salt高级命令的执行顺序是：
执行高级状态，默认去执行base环境的top.sls文件。
salt 会按照top.sls顺序执行base模块和prod模块定义的内容，此处，会根据指定的主机加载对应的文件，如果是base模块就执行 /srv/salt/base目录中指定的文件，并以这个目录为base环境的起始目录，如果是prod模块就去执行/srv/salt/prod目录中的对应文件，并以/srv/salt/prod目录为prod环境的起始目录。这里的base环境和prod环境路径都是在salt自身的配置文件中定义的。
执行prod环境中cluster目录中的haproxy-outside-keepalived.sls文件，此文件又include了一个modules/keepalived/install.sls的文件，所以会先执行prod环境中modules/keepalived/install.sls文件。
install.sls会完成编译安装初始化keepalived的任务，完成之后，再执行haproxy-outside-keepalived.sls文件。
haproxy-outside-keepalived.sls文件中主要负责了对不同业务上服务配置的修改。modules目录中存放的则是服务通用的基本安装配置。
SaltStack部署Memcached
部署软件包
下载部署软件包和初始配置文件，由于memcachd不需要使用配置文件来管理，所有只需要将对于的软件包放入files目录即可：
[root@node1 /srv/salt/prod/modules/memcached/files]# ls memcached-1.4.24.tar.gz
定义安装配置sls文件
[root@node1 /srv/salt/prod/modules/memcached]# cat install.sls include: - modules.libevent.install memcached-source-install: file.managed: - name: /usr/local/src/memcached-1.4.24.tar.gz - source: salt://modules/memcached/files/memcached-1.4.24.tar.gz - user: root - group: root - mode: 644 cmd.run: - name: cd /usr/local/src && tar zxf memcached-1.4.24.tar.gz && cd memcached-1.4.24&& ./configure --prefix=/usr/local/memcached --enable-64bit --with-libevent=/usr/local/libevent && make && make install - unless: test -d /usr/local/memcached # 存在此目录则不执行 - require: - cmd: libevent-source-install - file: memcached-source-install
由于需要安装libevent包，所以在安装之前还需先确认安装好libevent:
[root@node1 /srv/salt/prod/modules/libevent]# cat install.sls libevent-source-install: file.managed: - name: /usr/local/src/libevent-2.0.22-stable.tar - source: salt://modules/libevent/files/libevent-2.0.22-stable.tar - user: root - group: root - mode: 644 cmd.run: - name: cd /usr/local/src && tar xf libevent-2.0.22-stable.tar && cd libevent-2.0.22-stable && ./configure --prefix=/usr/local/libevent && make && make install - unless: test -d /usr/local/libevent - require: - file: libevent-source-install
libevent对应的安装包：
[root@node1 /srv/salt/prod/modules/libevent/files]# ls libevent-2.0.22-stable.tar
配置启动服务
将服务启动文件放到一个单独的bbs目录，作为一个与业务相关的配置模块：
[root@node1 /srv/salt/prod/bbs]# cat memcached.sls include: - moudles.memcached.install - moudles.user.www memcached-service: cmd.run: - name: /usr/local/memcached/bin/memcached -d -m 128 -p 11211 -c 8096 -u www - unless: netstat -ntlp | grep 11211 # 支持多次执行salt，已经启动就不再启动 - require: - cmd: memcached-source-install - user: www-user-group
这里定义了一个统一的用户,用于系统的标准化管理：
[root@node1 /srv/salt/prod/modules/user]# cat www.sls www-user-group: group.present: #用户组状态模块，添加用户，并授予id 1000,防止不同服务器id不同 - name: www - gid: 1000 user.present: #用户状态模块，指定用户信息 - name: www - fullname: www - shell: /sbin/nologin - uid: 1000 - gid: 1000
修改top file 运行高级状态
修改top file，指定memcached的安装:
# cat /srv/salt/base/top.sls base: '*': - init.init prod: 'node*': - cluster.haproxy-outside - cluster.haproxy-outside-keepalived 'node2': - bbs.memcached
运行高级状态：
# salt '*' state.highstate test=true # salt '*' state.highstate
确认memcached是否启动：
# telnet 172.16.10.61 11211 Trying 172.16.10.61... Connected to 172.16.10.61. Escape character is '^]'.
SaltStack部署NGINX-PHP
在php中配置memcached模块只需要在php.ini加上两行参数：
session.save_handler = memcached
session.save_path = "localhost:11211"
定义安装配置PHP sls文件
在/srv/salt/prod/modules/php目录中定义install.sls文件：
[root@node1 /srv/salt/prod/modules/php]# cat install.sls include: - modules.pkg.make - modules.user.www pkg-php: pkg.installed: - names: - mariadb-devel - swig - libjpeg-turbo - libjpeg-turbo-devel - libpng - libpng-devel - freetype - freetype-devel - libxml2 - libxml2-devel - zlib - zlib-devel - libcurl - libcurl-devel php-source-install: file.managed: - name: /usr/local/src/php-5.6.9.tar.gz - source: salt://modules/php/files/php-5.6.9.tar.gz - user: root - group: root - mode: 755 cmd.run: - name: cd /usr/local/src && tar zxf php-5.6.9.tar.gz && cd php-5.6.9&& ./configure --prefix=/usr/local/php-fastcgi --with-pdo-mysql=mysqlnd --with-mysqli=mysqlnd --with-mysql=mysqlnd --with-jpeg-dir --with-png-dir --with-zlib --enable-xml --with-libxml-dir --with-curl --enable-bcmath --enable-shmop --enable-sysvsem --enable-inline-optimization --enable-mbregex --with-openssl --enable-mbstring --with-gd --enable-gd-native-ttf --with-freetype-dir=/usr/lib64 --with-gettext=/usr/lib64 --enable-sockets --with-xmlrpc --enable-zip --enable-soap --disable-debug --enable-opcache --enable-zip --with-config-file-path=/usr/local/php-fastcgi/etc --enable-fpm --with-fpm-user=www --with-fpm-group=www && make && make install - require: - file: php-source-install - user: www-user-group - unless: test -d /usr/local/php-fastcgi pdo-plugin: cmd.run: - name: cd /usr/local/src/php-5.6.9/ext/pdo_mysql/ && /usr/local/php-fastcgi/bin/phpize && ./configure --with-php-config=/usr/local/php-fastcgi/bin/php-config && make&& make install - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/pdo_mysql.so - require: - cmd: php-source-install php-ini: file.managed: - name: /usr/local/php-fastcgi/etc/php.ini - source: salt://modules/php/files/php.ini-production - user: root - group: root - mode: 644 php-fpm: file.managed: - name: /usr/local/php-fastcgi/etc/php-fpm.conf - source: salt://modules/php/files/php-fpm.conf.default - user: root - group: root - mode: 644 php-fastcgi-service: file.managed: - name: /etc/init.d/php-fpm - source: salt://modules/php/files/init.d.php-fpm - user: root - group: root - mode: 755 cmd.run: - name: chkconfig --add php-fpm - unless: chkconfig --list | grep php-fpm - require: - file: php-fastcgi-service service.running: - name: php-fpm - enable: True - require: - cmd: php-fastcgi-service - watch: - file: php-ini - file: php-fpm
在php/files目录中，存放php的源码包和配置文件：
[root@node1 /srv/salt/prod/modules/php/files]# ll total 116216 -rw-r--r--. 1 root root 2362 Nov 17 12:33 init.d.php-fpm -rw-r--r--. 1 root root 118906880 Nov 17 11:06 php-5.6.28.tar -rw-r--r--. 1 root root 22252 Nov 17 12:34 php-fpm.conf.default -rw-r--r--. 1 root root 69599 Nov 17 12:33 php.ini-production
执行salt命令进行安装：
salt "*" state.sls modules.php.install saltenv=prod
提示：如果安装失败，可以查看错误是在那一步报错，对应的目录看文件是否分发，解压命令或编译安装命令是否正确等。
定义安装nginx sls文件
在/srv/salt/prod/modules/nginx目录中编写安装sls文件：
[root@node1 /srv/salt/prod/modules/nginx]# cat install.sls include: - modules.user.www nginx-source-install: file.managed: - name: /usr/local/src/nginx-1.10.2.tar.gz - source: salt://modules/nginx/files/nginx-1.10.2.tar.gz - user: root - group: root - mode: 755 cmd.run: - name: cd /usr/local/src && tar zxf nginx-1.10.2.tar.gz && cd nginx-1.10.2&& ./configure --prefix=/usr/local/nginx --user=www --group=www --with-http_ssl_module --with-http_stub_status_module --with-file-aio --with-http_dav_module --with-pcre && make && make install && chown -R www:www /usr/local/nginx - unless: test -d /usr/local/nginx - require: - user: www-user-group - file: nginx-source-install - pkg: make-pkg
nginx服务启动文件配置sls:
[root@node1 /srv/salt/prod/modules/nginx]# cat service.sls include: - modules.nginx.install nginx-init: file.managed: - name: /etc/init.d/nginx - source: salt://modules/nginx/files/nginx-init - mode: 755 - user: root - group: root cmd.run: - name: chkconfig --add nginx - unless: chkconfig --list | grep nginx - require: - file: nginx-init /usr/local/nginx/conf/nginx.conf: file.managed: - source: salt://modules/nginx/files/nginx.conf - user: www - group: www - mode: 644 nginx-service: service.running: - name: nginx - enable: True - reload: True - require: - cmd: nginx-init - watch: - file: /usr/local/nginx/conf/nginx.conf - file: nginx-online #目录内容改变则重新reload配置 nginx-offline: file.directory: - name: /usr/local/nginx/conf/vhost_offline #下线机器的配置文件存放处 nginx-online: file.directory: - name: /usr/local/nginx/conf/vhost_online # 创建vhost_online目录
拷贝安装的软件包和配置文件
在nginx/files目录下：
[root@node1 /srv/salt/prod/modules/nginx/files]# ll total 900 -rw-r--r-- 1 root root 910812 Nov 17 15:34 nginx-1.10.2.tar.gz -rw-r--r-- 1 root root 621 Nov 17 15:33 nginx.conf -rw-r--r-- 1 root root 2630 Nov 17 15:29 nginx-init
nginx默认配置文件，此文件全网统一，不同的业务配置在vhost_online目录：
[root@node1 /srv/salt/prod/modules/nginx/files]# cat nginx.conf
user www;
worker_processes 16; #生产实践
error_log logs/error.log error;
worker_rlimit_nofile 30000;
pid logs/nginx.pid;
events {
use epoll;
worker_connections 65535;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
tcp_nopush on;
underscores_in_headers on;
keepalive_timeout 10;
send_timeout 60;
include /usr/local/nginx/conf/vhost_online/*.conf; #此处对不同的业务定义不同的配置文件
server {
listen 8080;
server_name 127.0.0.1;
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
}
}
如果出现nginx 启动状态报错，或者启动失败的情况，先查看PID是否是在正确的位置，在启动脚本中修改pid file的文件路径。
不同业务配置
在/prod/bbs目录定义了对不同的业务所用的nginx配置文件：
[root@node1 /srv/salt/prod/bbs]# cat bbs.sls include: - modules.php.install - medules.php.php-memcached # 添加memchached模块 - medules.php.php-redis # 添加redis模块 - modules.nginx.service web-bbs: file.managed: - name: /usr/local/nginx/conf/vhost_online/bbs.conf - source: salt://bbs/files/nginx-bbs.conf - user: root - group: root - mode: 644 - require: - service: php-fastcgi-service - watch_in: - service: nginx-service
在bbs目录中：
[root@node1 /srv/salt/prod/bbs]# tree . ├── bbs.sls ├── files │ └── nginx-bbs.conf └── memcached.sls
添加memcache和redis缓存模块
将对应的模块文件拷贝到files目录下：
[root@node1 /srv/salt/prod/modules/php/files]# ll total 116384 -rw-r--r--. 1 root root 2362 Nov 17 12:33 init.d.php-fpm -rw-r--r-- 1 root root 36459 Nov 17 19:27 memcache-2.2.7.tgz -rw-r--r--. 1 root root 118906880 Nov 17 11:06 php-5.6.28.tar -rw-r--r-- 1 root root 22255 Nov 17 15:16 php-fpm.conf.default -rw-r--r--. 1 root root 69599 Nov 17 12:33 php.ini-production -rw-r--r-- 1 root root 134340 Nov 17 19:27 redis-2.2.7.tgz
对应的memcahed和redis安装sls:
[root@node1 /srv/salt/prod/modules/php]# cat php-memcache.sls memcache-plugin: file.managed: - name: /usr/local/src/memcache-2.2.7.tgz - source: salt://modules/php/files/memcache-2.2.7.tgz - user: root - group: root - mode: 755 cmd.run: - name: cd /usr/local/src && tar zxf memcache-2.2.7.tgz && cd memcache-2.2.7&& /usr/local/php-fastcgi/bin/phpize && ./configure --enable-memcache --with-php-config=/usr/local/php-fastcgi/bin/php-config && make&& make install - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/memcache.so require: - file: memcache-plugin - cmd: php-install memcache-php-config: file.append: - name: /usr/local/php-fastcgi/etc/php.ini - text: - extension=memcache.so
[root@node1 /srv/salt/prod/modules/php]# cat php-redis.sls redis-plugin: file.managed: - name: /usr/local/src/redis-2.2.7.tgz - source: salt://modules/php/files/redis-2.2.7.tgz - user: root - group: root - mode: 755 cmd.run: - name: cd /usr/local/src && tar zxf redis-2.2.7.tgz && cd redis-2.2.7&& /usr/local/php-fastcgi/bin/phpize && ./configure --with-php-config=/usr/local/php-fastcgi/bin/php-config && make&& make install - unless: test -f /usr/local/php-fastcgi/lib/php/extensions/*/redis.so require: - file: redis-plugin - cmd: php-install redis-php-config: file.append: - name: /usr/local/php-fastcgi/etc/php.ini - text: - extension=redis.so
定义高级状态
修改top file,添加nginx和php执行模块：
[root@node1 /srv/salt/base]# cat top.sls base: '*': - init.init prod: 'node*': - cluster.haproxy-outside - cluster.haproxy-outside-keepalived - bbs.bbs # 执行php nginx 模块 'node2': - bbs.memcached
执行高级状态：
# salt '*' state.highstate test=true
# salt '*' state.highstate
当执行salt 命令中，出现中断或者使用 ctrl+c 中断，可以通过jid查看进程的执行的状态：
#salt-run jobs.lookup_jid 20161118141146222666 查看salt ID执行的状态结果，即时执行结束，仍然可以查看到
# salt '*' saltutil.running 列出当前所有正在执行的job
[root@node1 /srv/salt/prod/modules/php]# salt '*' saltutil.running node1: |_ ---------- arg: fun: state.highstate jid: 20161118143331422864 #JID pid: 21329 ret: tgt: * tgt_type: glob user: root node2: |_ ---------- arg: fun: state.highstate jid: 20161118143331422864 #JID pid: 1665 ret: tgt: * tgt_type: glob user: root
# salt '*' saltutil.kill_job 20161118141146222666 结束进程