日均百万PV架构第一弹(基本架构搭建)

规划图:

以上架构可在生产环境中可以做出多样性的拆分与组合,文中四台主机限于虚拟机资源受限情况

规划如下: (结合图片四台主机的区分应该不难)

   slave1.king.com
       172.16.43.1
       DNS轮询 -> slave1.king.com , slave2.king.com
       haproxy七层代理流量(文件类型)分离 -> imgs , text , dynamic
       keepalived 为haproxy HA

   slave2.king.com
       172.16.43.2
       haproxy七层代理流量(文件类型)分离 -> imgs , text , dynamic
       keepalived 为haproxy HA

   slave3.king.com
       172.16.43.3
       nginx虚拟主机组 -> imgs1.king.com , imgs2.king.com
                        text1.king.com , text2.king.com
                        dynamic1.king.com
       php-fpm模块
       mysql数据库
       nfs文件共享  /nfsshared

   slave4.king.com
       172.16.43.4
       nginx虚拟主机 -> dynamic2.king.com
       php-fpm模块
       mysql-proxy(mmm)
       mysql数据库

过程如下:
   1. 一台haproxy(lvs)分离数据, dns解析双haproxy    (slave1)
   2. 四个虚拟主机分离imgs,text 静态测试                  (slave3)
   3. nginx , php-fpm , mysql 构建,测试动态代理        (slave3 , slave4)
   4. mysql-proxy 实现读写分离                                  (slave4)
   5. 实现双主keepalived HA的haproxy                      (slave2 , slave1)

1. 一台haproxy(lvs)分离数据, dns解析双haproxy (slave1.king.com)

i) dns解析
yum -y install named
.
vim /etc/named.rfc1912.zones
    zone "king.com" IN {
        type master;
        file "king.com.zone";
    };
.
vim /var/named/king.com.zone
$TTL 600
@       IN      SOA     dns.king.com.   adminmail.king.com. (
                        2014050401
                        1H
                        5M
                        3D
                        12H )
        IN      NS      dns
dns     IN      A       172.16.43.1
www     IN      A       172.16.43.88
www     IN      A       172.16.43.188
.
# 启动named
service named start

ii) haproxy配置
#安装 yum -y install haproxy
#
vim /etc/haproxy/haproxy.cfg
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
#
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 30000
#
listen stats
    mode http
    bind 0.0.0.0:8080
    stats enable
    stats hide-version
    stats uri     /haproxyadmin?stats
    stats realm   Haproxy\ Statistics
    stats auth    admin:admin
    stats admin if TRUE
#
frontend http-in
    bind *:80
    mode http
    log global
    option httpclose
    option logasap
    option dontlognull
    capture request  header Host len 20
    capture request  header Referer len 60
    acl img_static       path_beg       -i /p_w_picpaths /imgs
    acl img_static       path_end       -i .jpg .jpeg .gif .png
    acl text_static      path_beg       -i / /static /js /css
    acl text_static      path_end       -i .html .shtml .js .css
#
    use_backend img_servers             if img_static
    use_backend text_servers            if text_static
    default_backend dynamic_servers
#
backend img_servers
    balance roundrobin
    server imgsrv1 imgs1.king.com:80 check maxconn 4000
    server imgsrv2 imgs2.king.com:80 check maxconn 4000
#
backend text_servers
    balance roundrobin
    server textsrv1 text1.king.com:80 check maxconn 4000
    server textsrv2 text2.king.com:80 check maxconn 4000
#
backend dynamic_servers
    balance roundrobin
    server websrv1 dynamic1.king.com:80 check maxconn 1000
    server websrv2 dynamic2.king.com:80 check maxconn 1000

iii) hosts配置
#vim /etc/hosts
172.16.43.1 slave1.king.com
172.16.43.2 slave2.king.com
172.16.43.3 slave3.king.com
172.16.43.4 slave4.king.com
#
172.16.43.3 imgs1.king.com
172.16.43.3 imgs2.king.com
172.16.43.3 text1.king.com
172.16.43.3 text2.king.com
172.16.43.3 dynamic1.king.com
172.16.43.4 dynamic2.king.com
#
#
# 启动haproxy
service haproxy start

2. 四个虚拟主机分离imgs,text 静态测试 (slave3.king.com)

i) nginx 安装
   参见 http://apprentice.blog.51cto.com/2214645/1403422#nginx_install
ii) 配置 hosts 文件实现解析与 slave1.king.com hosts文件一致
iii) 配置nginx

#vim /etc/nginx/nginx.conf
#
#user  nobody;
worker_processes  1;
#
#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;
#
#pid        logs/nginx.pid;
#
events {
    worker_connections  1024;
}
#
http {
    include       mime.types;
    default_type  application/octet-stream;
#
    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$st,,atus $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';
#
    #access_log  logs/access.log  main;
#
    sendfile        on;
    #tcp_nopush     on;
#
    #keepalive_timeout  0;
    keepalive_timeout  5;
#
    gzip  on;
#
    open_file_cache          max=10000 inactive=60s;
    open_file_cache_valid    60s;
    open_file_cache_min_uses 2;
    open_file_cache_errors   on;
#
    proxy_cache_path /var/log/cache levels=1:2 keys_zone=web:100m max_size=1g inactive=12h;
#
    server {
        listen  80;
        server_name dynamic1.king.com;
#
        access_log  /var/log/nginx/dynamic1.access.log;
#
        location ~ \.php$ {
            root        /nfsshared/html;
            fastcgi_pass    172.16.43.3:9000;
            fastcgi_index   index.php;
            fastcgi_param   SCRIPT_FILENAME $document_root$fastcgi_script_name;
            include     fastcgi_params;
        }
#
        location / {
            root    /nfsshared/html;
            proxy_cache web;
            index   index.php index.html index.htm;
        }
    }
#
    server {
        listen       80;
        server_name  imgs1.king.com;
#
        access_log  /var/log/nginx/imgs1.access.log;
#
        location ~* \.(jpg|png|gif|jpeg)$ {
            root /nfsshared/html/imgs;
        }
#
        location ~* \.(jpg|png|gif|jpeg)$ {
            root /nfsshared/html/p_w_picpaths;
        }
#
            error_page  404              /404.html;
#
            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root   html;
            }
        }
#
        server {
            listen       80;
            server_name  imgs2.king.com;
#
            access_log  /var/log/nginx/imgs2.access.log;
#
        location ~* \.(jpg|png|gif|jpeg)$ {
            root /nfsshared/html/imgs;
        }
#
        location ~* \.(jpg|png|gif|jpeg)$ {
            root /nfsshared/html/p_w_picpaths;
        }
#
        error_page  404              /404.html;
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
#
    server {
        listen       80;
        server_name  text1.king.com;
#
        access_log  /var/log/nginx/text1.access.log;
#
        location ~* \.html$ {
            root /nfsshared/html/static;
        }
#
        location ~* \.css$ {
            root /nfsshared/html/css;
        }
#
        location ~* \.js$ {
            root /nfsshared/html/js;
        }
#
            location / {
                root   /nfsshared/html/static;
                index  index.html index.htm;
            }
#
            error_page  404              /404.html;
#
            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root   html;
            }
        }
#
    server {
        listen       80;
        server_name  text2.king.com;
#
        access_log  /var/log/nginx/text2.access.log;
#
        location ~* \.html$ {
            root /nfsshared/html/static;
        }
#
        location ~* \.css$ {
            root /nfsshared/html/css;
        }
#
        location ~* \.js$ {
            root /nfsshared/html/js;
        }
#
        location / {
            root   /nfsshared/html/static;
            index  index.html index.htm;
        }
#
        error_page  404              /404.html;
#
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

iv) 测试文件

vim /nfsshared/static/index.html






        测试文字

                  # vim /nfsshared/css/test.css #testid {     font-size: 40px;     border: 10px red dashed; } # vim /nfsshared/js/test.js window.onload = function() {     alert("加载页面...使用js.."); }

v) 测试

访问 imgs2.king.com/2.jpg  text1.king.com/test.css 均无问题

3. nginx , php-fpm , mysql 构建 (slave3.king.com , slave4.king.com)

#i) nginx安装及配置
#
vim /etc/nginx/nginx.conf
server {
    listen       80;
    server_name  dynamic2.king.com;
#
    access_log  /var/log/nginx/dynamic2.access.log;
    open_file_cache          max=10000 inactive=60s;
    open_file_cache_valid    60s;
    open_file_cache_min_uses 2;
    open_file_cache_errors   on;
#
    proxy_cache_path /var/log/cache levels=1:2 keys_zone=web:100m max_size=1g inactive=12h;
    location ~ \.php$ {
        root           /nfsshared/html;
        fastcgi_pass   172.16.43.4:9000;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include        fastcgi_params;
    }
#
    location / {
        root   /nfsshared/html;
        proxy_cache web;
        index  index.php index.html index.htm;
    }
}

ii) 解决php安装依赖
yum -y groupinstall "Desktop Platform Development"
yum -y install libmcrypt-devel
yum -y install bzip2-devel
#
ii) 安装php with fpm
tar xf php-5.4.19.tar.bz2
cd php-5.4.19
./configure --prefix=/usr/local/php --with-mysql=mysqlnd --with-pdo-mysql=mysqlnd --with-mysqli=mysqlnd --with-openssl --enable-mbstring --with-freetype-dir --with-jpeg-dir --with-png-dir --with-zlib --with-libxml-dir=/usr --enable-xml --enable-sockets --enable-fpm --with-mcrypt --with-config-file-path=/etc --with-config-file-scan-dir=/etc/php.d --with-bz2
make && make install
#
ii) 为php提供配置文件：
cp php.ini-production /etc/php.ini
#
ii) 配置php-fpm
# 为php-fpm提供SysV init脚本，并将其添加至服务列表：sapi在源码包下
cp sapi/fpm/init.d.php-fpm  /etc/rc.d/init.d/php-fpm
chmod +x /etc/rc.d/init.d/php-fpm
chkconfig --add php-fpm
chkconfig php-fpm on
#
# 为php-fpm提供配置文件：
cp /usr/local/php/etc/php-fpm.conf.default /usr/local/php/etc/php-fpm.conf
# 编辑php-fpm的配置文件：
vim /usr/local/php/etc/php-fpm.conf
# 配置fpm的相关选项为你所需要的值，并启用pid文件：
pid = /usr/local/php/var/run/php-fpm.pid
listen = 172.16.43.2:9000

iii) MariaDB安装与配置(创建mysql的数据目录)
mkdir /data
groupadd -r mysql
useradd -g mysql -r -s /sbin/nologin -M -d /data mysql
chown -R mysql:mysql /data
#
iii) 安装二进制mysql
tar xf mysql-5.5.33-linux2.6-x86_64.tar.gz -C /usr/local
cd /usr/local
ln -sv mysql-5.5.33-linux2.6-x86_64 mysql
cd mysql
chown -R mysql:mysql  .
mysql/scripts/mysql_install_db --user=mysql --datadir=/data
chown -R root  .
# 提供mysql的配置文件
cp support-files/my-large.cnf  /etc/my.cnf
# 需要添加如下行指定mysql数据文件的存放位置：
datadir = /data
#
iii) 为mysql提供sysv服务脚本：
cd /usr/local/mysql
cp support-files/mysql.server  /etc/rc.d/init.d/mysqld
chmod +x /etc/rc.d/init.d/mysqld
添加至服务列表：
chkconfig --add mysqld
chkconfig mysqld on
echo "export PATH=/usr/local/mysql/bin:$PATH" > /etc/profile.d/mysql.sh
. /etc/profile.d/mysql.sh
#
iii) 启动服务并授权php服务器账号访问
service mysqld restart
mysql
grant all on *.* to 'root'@'172.16.%.%' identified by '123456';
flush privileges;

slave4.king.com也是如上的配置 nginx , php-fpm , mysql

   其他配置参见 http://apprentice.blog.51cto.com/2214645/1403422#php_install

4. mysql-proxy(mmm) 实现读写分离 (slave4.king.com)
   其他配置参见 http://apprentice.blog.51cto.com/2214645/1399141#mysql_mmm

5. 实现双主keepalived HA的haproxy (slave2.king.com, slave1.king.com)
   其他配置参见 http://apprentice.blog.51cto.com/2214645/1404853#keepalivedHA

可用性测试,在slave2.king.com瘫痪后,vip转移至slave1.king.com一个节点上