七层负载均衡——nginx

一.nginx源码下载及配置

1.nginx官网下载地址

官方地址：https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/src
下载地址：wget https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/get/master.tar.gz

[root@server1 ~]# ls
nginx-1.20.1.tar.gz
[root@server1 ~]# tar -zxvf nginx-1.20.1.tar.gz
[root@server1 ~]# cd nginx-1.20.1/  ##注意：需要在此目录下执行configure
[root@server1 nginx-1.20.1]# ls
auto     CHANGES.ru  configure  html     man     src
CHANGES  conf        contrib    LICENSE  README
[root@server1 nginx-1.20.1]# ./configure --help  ##可以看到可用参数

2. 进入nginx源码目录进行编译

[root@server1 nginx-1.20.1]#  ./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module 
checking for OS
 + Linux 3.10.0-957.el7.x86_64 x86_64
checking for C compiler ... not found

./configure: error: C compiler cc is not found   ##提示没有下载编辑器，需要下载编辑

[root@server1 nginx-1.20.1]# yum install -y gcc

提示缺少pcre库，下载pcre
[root@server1 nginx-1.20.1]# yum install -y pcre-devel    ##下载库通常为库名+ -devel

  提示缺少OpenSSL库，下载OpenSSL

[root@server1 nginx-1.20.1]# ./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module     ##安装模块

安装好模块之后，会生成Makefile文件，然后使用make指令

make指令，是在安装有GNU Make的计算机上的可执行指令。该指令是读入一个名为 ’Makefile’的文件，然后执行这个文件中指定的指令。

 make完成之后，使用make install 命令将可执行文件、第三方依赖包和文档复制到正确的路径

 此时安装完成，可以看到objs目录下有nginx，conf/中放有配置文件

 创建软链接到/usr/local/sbin/（这样在开启nginx服务的时候就不需要进到目录下开启，软连接的方式可以方便开启全局nginx）

[root@server1 ~]# /usr/local/nginx/sbin/nginx -t   ##查看是否可执行
[root@server1 ~]# ln -s /usr/local/nginx/sbin/nginx /usr/local/bin/  ##建立软连接
[root@server1 ~]# which nginx
/usr/local/bin/nginx
[root@server1 ~]# nginx             ##启动nginx
[root@server1 ~]# nginx -s reload
[root@server1 ~]# nginx -s stop
[root@server1 ~]# nginx
[root@server1 ~]# curl localhost

 去访问http://172.25.52.1 

若是想使用systemctl 命令需要添加脚本，将nginx放进指定文件中

[root@server1 nginx]# cd /usr/lib/systemd/system/
[root@server1 system]# vim nginx.service   

[Unit]
Description=The NGINX HTTP and reverse proxy server
After=syslog.target network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
PIDFile=/usr/local/nginx/logs/nginx.pid
ExecStartPre=/usr/local/nginx/sbin/nginx -t
ExecStart=/usr/local/nginx/sbin/nginx
ExecReload=/usr/local/nginx/sbin/nginx -s reload
ExecStop=/bin/kill -s QUIT $MAINPID
PrivateTmp=true
[root@server1 system]# cd /usr/local/nginx/logs/  ##日志文件目录
[root@server1 logs]# ls
access.log  error.log  nginx.pid
[root@server1 logs]# cd /usr/local/nginx/
[root@server1 nginx]# cd sbin/
[root@server1 sbin]# ls
nginx
[root@server1 nginx]# cd /usr/lib/systemd/system
[root@server1 system]# systemctl daemon-reload  ##刷新服务列表

[root@server1 system]# systemctl start nginx.service  ##如果启动失败，kill所有nginx的进程
[root@server1 system]# ps ax| grep nginx
21142 pts/0    S+     0:00 grep --color=auto nginx
[root@server1 system]# systemctl start nginx.service
[root@server1 system]# systemctl status nginx.service 

二、优化配置

编辑主配置文件，优化nginx的工作效率

1.修改进程数

[root@server1 system]# nginx -s stop
[root@server1 system]# cd /usr/local/nginx/conf/
[root@server1 conf]# ls
[root@server1 conf]# vim nginx.conf
user  nginx;
worker_processes  2;
worker_cpu_affinity 01 10;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

[root@server1 conf]# ps ax

 

2.修改open files值

可以看到此时的open files数为1024

 想将open files值改为65535，需要两步：
1、修改主配置文件
2、修改文件/etc/security/limits.conf
vim /etc/security/limits.conf

nginx         -      nofile         65535

为安全和方便，我们为nginx添加一个用户

[root@server1 conf]# useradd nginx

[root@server1 conf]# usermod -s /sbin/nologin nginx    ##一般管理软件的用户不需要登陆
[root@server1 conf]# id nginx
uid=1000(nginx) gid=1000(nginx) groups=1000(nginx)
[root@server1 conf]# nginx
[root@server1 conf]# ps axu

 切换用户 查询可看到open files 值更改为65535

3.在主配置文件添加负载均衡器和反向代理

 vim nginx.conf

[root@server1 conf]# vim nginx.conf
user  nginx nginx;
worker_processes  2;
worker_cpu_affinity 01 10;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  65535;
}


http {
	upstream westos{
	server 172.25.52.2:80;
	server 172.25.52.3:80;
	server 172.25.52.1:8080 backup;
	}
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  www.westos.org;

        location / {
           proxy_pass http://westos;       ##反向代理
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
[root@server1 conf]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@server1 conf]# nginx -s reload
[root@server1 conf]# vim /etc/httpd/conf/httpd.conf
Listen 8080
[root@server1 conf]# systemctl start httpd
[root@server1 conf]# curl localhost:8080
server1

 

在客户端访问：

在真机中进行地址解析，curl可以看到负载均衡

[root@foundation52 ~]# vim /etc/hosts 
172.25.52.1 server1  www.westos.org

[root@server2 ~]# systemctl stop httpd
[root@server3 ~]# systemctl stop httpd

[root@foundation52 ~]# for i in {1..10}; do curl www.westos.org;done ##将server2和3的httpd关闭

[root@server3 ~]# systemctl stop httpd
[root@server2 ~]# systemctl start httpd

[root@foundation52 ~]# for i in {1..10}; do curl www.westos.org;done  ##server3关闭,server2打开

[root@server2 ~]# systemctl stop httpd
[root@server3 ~]# systemctl start httpd

[root@foundation52 ~]# for i in {1..10}; do curl www.westos.org;done         ##server3开,server2关闭

[root@server3 ~]# systemctl start httpd
[root@server2 ~]# systemctl start httpd

[root@foundation52 ~]# for i in {1..10}; do curl www.westos.org;done  ##server2和server3都开

三.第三方模块sticky

sticky模块与Ip_hash都是与负载均衡算法相关，但又有差别，差别是：

1.ip hash，根据客户端的IP，将请求分配到不同的服务器上
2.sticky，根据服务器给客户端的cookie，客户端再次请求时会带上此cookie，nginx会把有此cookie的请求转发到颁发cookie的服务器上

注意：在一个局域网内有3台电脑，他们有3个内网IP，但是他们发起请求时，却只有一个外网IP，是电信运营商分配在他们连接那个路由器上的，如果使用 ip_hash 方式，则Nginx会将请求分配到不同上游服务器，如果使用 sticky 模块，则会把请求分配到办法cookie的服务器上，实现：内网nat用户的均衡。这是iphash无法做到的。
工作原理：

Sticky是基于cookie的一种负载均衡解决方案，通过分发和识别cookie，使来自同一个客户端的请求落在同一台服务器上，默认cookie标识名为route ：

1.客户端首次发起访问请求，nginx接收后，发现请求头没有cookie，则以轮询方式将请求分发给后端服务器。
2.后端服务器处理完请求，将响应数据返回给nginx。
3.此时nginx生成带route的cookie，返回给客户端。route的值与后端服务器对应，可能是明文，也可能是md5、sha1等Hash值
4.客户端接收请求，并保存带route的cookie。
5.当客户端下一次发送请求时，会带上route，nginx根据接收到的cookie中的route值，转发给对应的后端服务器。

[root@server1 ~]# ls
nginx-1.20.1  nginx-1.20.1.tar.gz  nginx-goodies-nginx-sticky-module-ng-08a395c66e42.zip
[root@server1 ~]# yum install -y unzip   ##下载unzip解压软件，用unzip命令解压
[root@server1 ~]# unzip nginx-goodies-nginx-sticky-module-ng-08a395c66e42.zip

 

1.关闭debug 

[root@server1 ~]# cd nginx-1.20.1/
[root@server1 nginx-1.20.1]# vim auto/cc/gcc
# stop on warning
CFLAGS="$CFLAGS -Werror"

# debug
#CFLAGS="$CFLAGS -g"   ##注释

2.隐藏版本号

[root@server1 nginx-1.20.1]# vim src/core/nginx.h 

#define NGINX_VER          "nginx"       ##将后面的版本号删除

3.停止nginx

[root@server1 nginx-1.20.1]# nginx -s stop
[root@server1 nginx-1.20.1]# ps aux| grep nginx
[root@server1 nginx-1.20.1]# cd nginx-1.20.1/objs/
[root@server1 objs]# du -h nginx
3.8M    nginx

4.加入模块：

[root@server1 nginx-1.20.1]# make clean
rm -rf Makefile objs 
[root@server1 nginx-1.20.1]# ./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --add-module=/root/nginx-goodies-nginx-sticky-module-ng-08a395c66e42  ##添加模块
root@server1 nginx-1.20.1]# make      ##第二步
[root@server1 nginx-1.20.1]# cd objs/
[root@server1 objs]# ls
addon         Makefile  nginx.8            ngx_auto_headers.h  ngx_modules.o
autoconf.err  nginx     ngx_auto_config.h  ngx_modules.c       src
[root@server1 objs]# du -h nginx
936K	nginx
[root@server1 objs]# cd /usr/local/nginx/sbin/
[root@server1 sbin]# ls
nginx
[root@server1 sbin]# mv nginx nginx.old  ##将之前的备份
[root@server1 sbin]# ls
nginx.old
[root@server1 sbin]# cd -
/root/nginx-1.20.1/objs
[root@server1 objs]# cp nginx /usr/local/nginx/sbin/
[root@server1 objs]# nginx
[root@server1 objs]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful

[root@server1 ~]# cd /usr/local/nginx/conf/
[root@server1 conf]# vim nginx.conf
user  nginx nginx;
worker_processes  2;
worker_cpu_affinity 01 10;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

events {
    worker_connections  65535;
}

http {
	upstream westos{
	sticky;                         ###添加sticky
	server 172.25.52.2:80;
	server 172.25.52.3:80;
	#server 172.25.52.1:8080 backup;  ##注释（不注释语法会出错）
	}
    include       mime.types;
    default_type  application/octet-stream;
[root@server1 conf]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@server1 conf]# nginx -s reload
[root@server1 conf]# curl localhost -I

[root@server1 conf]# vim nginx.conf                  
[root@server1 conf]# nginx -s reload

     http {
        upstream westos{
        #sticky;
        server 172.25.52.2:80 weight=2;     ##给server2加权重
        server 172.25.52.3:80;
        #server 172.25.52.1:8080 backup;
        }
[root@server1 conf]# nginx -s reload

在客户端访问查看：
[root@foundation52 kiosk]# for i in {1..10}; do curl www.westos.org;done   ##server2是3的两倍

 [root@server1 conf]# vim nginx.conf                 
http {
        upstream westos{
        ip_hash;               ##固定ip,固定负载
        #sticky;
        server 172.25.52.2:80;
        server 172.25.52.3:80;
        #server 172.25.52.1:8080 backup;
        }

[root@server1 conf]# nginx -s reload
[root@foundation52 kiosk]# for i in {1..10}; do curl www.westos.org;done 

四.平滑升级

平滑升级就是在不影响当前程序运行的情况下，对Nginx版本进行升级、添加/删除服务器模块。

步骤：
下载nginx新版本软件,正常执行./configure 和make 但不要执行make install
备份原程序
拷贝新程序
获取当前nginx主进程pid
升级新程序
关闭原worker进程但保留主进程:为了回退

建立链接：真实主机（下载不需要进行scp）

 [root@foundation52 kiosk]# iptables -t nat -I POSTROUTING -s 172.25.52.250/24 -j MASQUERADE

[root@server1 ~]# nginx
[root@server1 ~]# nginx -s stop
[root@server1 ~]# cd /usr/local/nginx/conf/
[root@server1 conf]# vim nginx.conf
http {
        upstream westos{
        #sticky;
        server 172.25.52.2:80 weight=2;
        server 172.25.52.3:80;
        #server 172.25.52.1:8080 backup;
        }

[root@server1 conf]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@server1 conf]# cd
[root@server1 ~]# cd /usr/local/nginx/sbin/
[root@server1 sbin]# ls
nginx  nginx.old
[root@server1 sbin]# rm -fr nginx ##将隐藏版本号的nginx文件删除
[root@server1 sbin]# mv nginx.old  nginx   ##还原原来的nginx文件
[root@server1 sbin]# ls
nginx
[root@server1 sbin]# nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@server1 sbin]# nginx
[root@server1 sbin]# curl localhost -I

[root@server1 ~]# tar zxf nginx-1.20.2.tar.gz
[root@server1 ~]# ls
nginx-1.20.1         nginx-1.20.2.tar.gz
nginx-1.20.1.tar.gz  nginx-goodies-nginx-sticky-module-ng-08a395c66e42
nginx-1.20.2         nginx-goodies-nginx-sticky-module-ng-08a395c66e42.zip
[root@server1 ~]# cd nginx-1.20.2
[root@server1 nginx-1.20.2]# ./configure --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module
[root@server1 nginx-1.20.2]# make
[root@server1 nginx-1.20.2]# cd /usr/local/nginx/sbin/
[root@server1 sbin]# ls
nginx
[root@server1 sbin]# cp nginx nginx.old  ##将20.1的版本复制为old
[root@server1 sbin]# ls
nginx  nginx.old

 

[root@server1 ~]# cd nginx-1.20.2/
[root@server1 nginx-1.20.2]# ls objs/
autoconf.err  nginx    ngx_auto_config.h   ngx_modules.c  src
Makefile      nginx.8  ngx_auto_headers.h  ngx_modules.o
[root@server1 nginx-1.20.2]# cd objs/
[root@server1 objs]# \cp -f nginx /usr/local/nginx/sbin/
[root@server1 objs]# cd /usr/local/nginx/sbin/
[root@server1 sbin]# ll
total 9716
-rwxr-xr-x 1 root root 6025432 Dec  9 17:33 nginx
-rwxr-xr-x 1 root root 3916992 Dec  9 17:30 nginx.old
[root@server1 sbin]# date
Thu Dec  9 17:33:42 CST 2021

[root@server1 sbin]# ps ax | grep nginx
[root@server1 sbin]# kill -USR2 3574
[root@server1 sbin]# ps ax | grep nginx
[root@server1 sbin]# curl -I localhost  

[root@server1 sbin]# kill -WINCH 3574
[root@server1 sbin]# curl -I localhost    ##已经换成显示1.20.2的版本

 

[root@server1 sbin]# ls
nginx  nginx.old
[root@server1 sbin]# cp nginx nginx.new
[root@server1 sbin]# \cp -f nginx.old nginx
[root@server1 sbin]# ll
[root@server1 sbin]# kill -HUP 3574                ##将旧的打开
[root@server1 sbin]# ps ax | grep nginx
[root@server1 sbin]# kill -WINCH 6217   ##关闭新的
[root@server1 sbin]# curl -I localhost