LVS+KeepAlived -->Squid -->Web Cluster

LVS+KeepAlived

(CentOS5.6 2.6.18-238.12.1.el5-x86_64)

[email protected]




VIP      :    192.168.0.6
DRM      :    eth0 192.168.0.4
DRB      :    eth0 192.168.0.5
node1    :    eth0 192.168.0.7
node1    :    eth1 10.0.0.1
node2    :    eth0 192.168.0.8
node2    :    eth1 10.0.0.2
node3    :    eth0 192.168.0.9
node3    :    eth1 10.0.0.3
Web1     :    eth0 10.0.0.8
Web2     :    eth0 10.0.0.9
Web3     :    eth0 10.0.0.10
Openfiler:    eth0 10.0.0.88

DRM:
[root@DRM ~]# cat lvs+keepalived.sh

#!/bin/bash
for i in lo all eth0
do
    for j in arp_ignore arp_announce
    do
        if [ $j = "arp_ignore" ]; then
             echo "1" >/proc/sys/net/ipv4/conf/${i}/${j}
        else
             echo "2" >/proc/sys/net/ipv4/conf/${i}/${j}
        fi
    done
done

sysctl -p > /dev/null 2>&1

yum -y install ipvsadm
tar -zxvf keepalived-1.1.20.tar.gz
cd keepalived-1.1.20
./configure --prefix=/usr --exec-prefix=/usr \
--bindir=/usr/bin --sbindir=/usr/sbin --sysconfdir=/etc --datadir=/usr/share \
--includedir=/usr/include --libdir=/usr/lib64 --libexecdir=/usr/libexec \
--localstatedir=/var --mandir=/usr/share/man --infodir=/usr/share/info \
--sharedstatedir=/usr/com \
--with-kernel-dir=/usr/src/kernels/2.6.18-238.12.1.el5-x86_64

#########################################
make && make install
cd ../
[root@DRM ~]# sh lvs+keepalived.sh
[root@DRM ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     complex.invoke@gmail.com
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.6
    }
}

virtual_server 192.168.0.6 80 {
    delay_loop 6
    lb_algo wrr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.0.7 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
    }
    real_server 192.168.0.8 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
}
    real_server 192.168.0.9 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
    }

}
[root@DRM ~]# chkconfig --level 345 keepalived on
[root@DRM ~]# service keepalived start
DRB:
[root@DRB ~]# sh lvs+keepalived.sh

[root@DRB ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     complex.invoke@gmail.com
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.6
    }
}

virtual_server 192.168.0.6 80 {
    delay_loop 6
    lb_algo wrr
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    real_server 192.168.0.7 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
    }
    real_server 192.168.0.8 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
}
    real_server 192.168.0.9 80 {
        weight 3
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
            }
    }

}
[root@DRB ~]# chkconfig --level 345 keepalived on

[root@DRB ~]# service keepalived start

node1 node2 node3:

[root@node1 ~]# cat node.sh

#!/bin/bash
VIP=192.168.0.6
. /etc/rc.d/init.d/functions
case $1 in
     start)
          ifconfig lo:0 $VIP netmask 255.255.255.255 broadcast $VIP
          route add -host $VIP dev lo:0
          for i in lo all eth0; do
              for j in arp_ignore arp_announce; do
                  if [ $j = "arp_ignore" ]; then
                      echo "1" >/proc/sys/net/ipv4/conf/${i}/${j}
                  else
                      echo "2" >/proc/sys/net/ipv4/conf/${i}/${j}
                  fi
              done
          done

          sysctl -p > /dev/null 2>&1
     ;;
     stop)
         ifconfig lo:0 down
         route del $VIP > /dev/null 2>&1
          for i in lo all eth0; do
              for j in arp_ignore arp_announce; do
                     echo "0" >/proc/sys/net/ipv4/conf/${i}/${j}
              done
          done

          sysctl -p > /dev/null 2>&1

     ;;
     *)
          echo "Usage: $(basename $0) {start|stop}"
          exit 1
     ;;
esac
[root@node1 ~]# sh node.sh start

[root@node1 ~]# cat squid-install.sh

#!/bin/bash
mkdir -p /home/squid/{cache,log}
ln -s /home/squid /var/spool/squid
yum -y install squid
chown squid:squid -R /home/squid
squid -z
squid -s
#crontab -e
#59 * * * * /usr/sbin/squid -k rotate
[root@node1 ~]# sh squid-install.sh

[root@node1 ~]# vim /etc/squid/squid.conf


acl all src 0.0.0.0/0.0.0.0
acl manager proto cache_object
acl localhost src 127.0.0.1/32
acl to_localhost dst 127.0.0.0/8
acl SSL_ports port 443
acl Safe_ports port 80 3130
acl CONNECT method CONNECT
acl purge method PURGE

http_access allow manager localhost
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny to_localhost
http_access allow purge localhost
http_access deny purge

# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS

http_access allow localhost
http_access allow all
http_reply_access allow all
icp_access allow all

miss_access allow all
ident_lookup_access deny all
reply_body_max_size 0 allow all
follow_x_forwarded_for deny all
acl_uses_indirect_client on
delay_pool_uses_indirect_client on
log_uses_indirect_client on

http_port 3128
http_port 80 vhost vport
icp_port 3130

cache_peer 10.0.0.8    parent  80 0    no-query originserver no-digest round-robin
cache_peer 10.0.0.9    parent  80 0    no-query originserver no-digest round-robin
cache_peer 10.0.0.10   parent  80 0    no-query originserver no-digest round-robin
cache_peer 192.168.0.7 sibling 80 3130    # replace 7 with 8 or 9 on node2 or node3
cache_peer 192.168.0.8 sibling 80 3130 proxy-only no-netdb-exchange
cache_peer 192.168.0.9 sibling 80 3130 proxy-only no-netdb-exchange

dead_peer_timeout 10 seconds
hierarchy_stoplist cgi-bin ?
acl QUERY urlpath_regex cgi-bin \?
cache deny QUERY
cache_mem 64 MB
maximum_object_size_in_memory 5120 KB
memory_replacement_policy lru
cache_replacement_policy heap LFUDA

cache_dir ufs /home/squid/cache 100 16 256

store_dir_select_algorithm round-robin
max_open_disk_fds 0
minimum_object_size 0 KB
maximum_object_size 20480 KB
cache_swap_low 85
cache_swap_high 95
logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %"%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh

access_log /home/squid/log/access.log squid
cache_log /home/squid/log/cache.log

cache_store_log none
logfile_rotate 10
log_ip_on_direct on

mime_table /etc/squid/mime.conf
pid_filename /var/run/squid.pid
strip_query_terms on
diskd_program /usr/lib64/squid/diskd-daemon
unlinkd_program /usr/lib64/squid/unlinkd
acl QUERY urlpath_regex cgi-bin \?
cache deny QUERY
refresh_pattern .php$      1440 20%  10080
refresh_pattern .          0    20%  4320

quick_abort_min 16 KB
quick_abort_max 16 KB
quick_abort_pct 95

read_ahead_gap 16 KB
negative_ttl 5 minutes
positive_dns_ttl 6 hours
negative_dns_ttl 1 minute
range_offset_limit 0 KB
minimum_expiry_time 60 seconds
store_avg_object_size 13 KB
store_objects_per_bucket 20
request_header_max_size 20 KB
reply_header_max_size 20 KB
request_body_max_size 0 KB
via on
cache_vary on
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
refresh_stale_hit 0 seconds
relaxed_header_parser on
forward_timeout 4 minutes
connect_timeout 1 minute
peer_connect_timeout 30 seconds
read_timeout 15 minutes
request_timeout 5 minutes
persistent_request_timeout 2 minutes
client_lifetime 1 day
pconn_timeout 1 minute
ident_timeout 10 seconds
shutdown_lifetime 30 seconds

cache_mgr expertshell@gmail.com

mail_program mail
cache_effective_user squid
cache_effective_group squid

visible_hostname node1

umask 027
minimum_direct_hops 4
minimum_direct_rtt 400
netdb_low 900
netdb_high 1000
netdb_ping_period 5 minutes
icp_query_timeout 2000
digest_generation on
log_icp_queries on
icp_hit_stale on
maximum_icp_query_timeout 2000
minimum_icp_query_timeout 5
incoming_http_average 4
incoming_dns_average 4
min_icp_poll_cnt 8
min_dns_poll_cnt 8
min_http_poll_cnt 8
tcp_recv_bufsize 0 bytes
balance_on_multiple_ip on
[root@node1 ~]# vim  nat-iptables.sh

#!/bin/bash
if (( UID != 0 )); then
   echo "permission denied, please run as root privilige!"
   exit 1
fi

# global settings
IPTABLES="/sbin/iptables"
MODPROBE="/sbin/modprobe"
INT_NET="10.0.0.0/8"
# Flush existing rules and set chain policy setting to DROP
for i in F X Z
do
    $IPTABLES -$i
    $IPTABLES -$i -t nat
done
for j in INPUT OUTPUT FORWARD
do
     $IPTABLES -P $j  ACCEPT
done

# load modules
modules="xt_state ipt_recent xt_tcpudp ipt_MASQUERADE iptable_filter ip_tables iptable_nat ip_nat_ftp ip_nat_irc ip_conntrack ip_conntrack_ftp ip_conntrack_irc"
  for mod in $modules
  do
        testmod=$( lsmod | grep "$mod" )
        if [ "x$testmod" = "x" ]; then
              $MODPROBE $mod
        fi
  done

# setting kernel network features
for k in ip_dynaddr tcp_syncookies icmp_echo_ignore_broadcasts
do      echo "1" > /proc/sys/net/ipv4/$k
done

pq="/proc/sys/net/ipv4/conf/*/"
qp="rp_filter log_martians"
bd="accept_source_route accept_redirects send_redirects"
for u in ${qp}
do
    for v in ${pq}${u}
    do
        echo "1" > $v
    done
done
for w in ${bd}
do
    for x in ${pq}${w}
    do
        echo "0" > $x
    done
done

$IPTABLES -t nat -A POSTROUTING -s $INT_NET -o eth0 -j MASQUERADE
# forwarding
sed -i 's#net.ipv4.ip_forward = 0#net.ipv4.ip_forward = 1#g' /etc/sysctl.conf
echo "1" > /proc/sys/net/ipv4/ip_forward

# save iptables policy:
  service ip6tables stop
  chkconfig ip6tables off
  chkconfig --level 345 iptables on
  iptables-save > /etc/sysconfig/iptables
# restore iptables policy:
# iptables-restore < /etc/sysconfig/iptables
[root@node1 ~]# sh nat-iptables.sh

Openfiler:

[root@Openfiler ~]# fdisk -l | grep ^/dev/sd

/dev/sdb1            2048    41943039    20970496   83  Linux

/dev/sdc1            2048    41943039    20970496   83  Linux

/dev/sdd1            2048    41943039    20970496   83  Linux

/dev/sde1            2048    41943039    20970496   83  Linux

/dev/sdf1            2048    41943039    20970496   83  Linux

/dev/sda1   *          63      208844      104391   83  Linux

/dev/sda2          208845     4401809     2096482+  83  Linux

/dev/sda3         4401810    16771859     6185025   83  Linux

/dev/sda4        16771860    20964824     2096482+   5  Extended

/dev/sda5        16771923    20964824     2096451   83  Linux

[root@Openfiler ~]# partprobe

[root@Openfiler ~]# mdadm --create --auto=yes /dev/md0 --level=5 \

  --raid-devices=4 --spare-devices=1 /dev/sd{b,c,d,e,f}1

[root@Openfiler ~]# mkfs.btrfs /dev/md0

WARNING! - Btrfs Btrfs v0.19 IS EXPERIMENTAL

WARNING! - see http://btrfs.wiki.kernel.org before using

fs created label (null) on /dev/md0

        nodesize 4096 leafsize 4096 sectorsize 4096 size 59.99GB

Btrfs Btrfs v0.19

[root@Openfiler ~]# mkdir /mnt/raid

[root@Openfiler ~]# mount /dev/md0 /mnt/raid

[root@Openfiler ~]# df -Th | grep /dev/md0

/dev/md0     btrfs     60G   56K   60G   1% /mnt/raid

[root@Openfiler ~]# mdadm --detail /dev/md0 | grep -i uuid

           UUID : bcb160c2:a8409638:71d19d46:74c61ecd

[root@Openfiler ~]# vim /etc/mdadm.conf

ARRAY /dev/md0 UUID=bcb160c2:a8409638:71d19d46:74c61ecd

[root@Openfiler ~]# vim /etc/fstab

/dev/md0           /mnt/raid             btrfs   defaults        1 2

[root@Openfiler ~]# chown apache:apache -R /mnt/raid

[root@Openfiler ~]# vim /mnt/raid/index.php



PHP



Server:


Path:
Time:


[root@Openfiler ~]# vim /etc/exports

/mnt/raid 10.0.0.0/24(rw,anonuid=48,anongid=48)

[root@Openfiler ~]# service portmap start

[root@Openfiler ~]# service nfs start

Web1 Web2 Web3:

[root@Web1 ~]# mkdir -p /usr/wwwroot/html

[root@Web1 ~]# showmount -e 10.0.0.88

Export list for 10.0.0.88:

/mnt/raid 10.0.0.0/24

[root@Web1 ~]# mount -t nfs -o rw,bg,intr,soft,rsize=32768,wsize=32768 10.0.0.88:/mnt/raid /usr/wwwroot/html

[root@Web1 ~]# yum -y install httpd mysql mysql-server php php-devel php-mysql

[root@Web1 ~]# sed -i 's#/var/www/html#/usr/wwwroot/html#g' /etc/httpd/conf/httpd.conf

[root@Web1 ~]# chown apache:apache -R /usr/wwwroot/html

[root@Web1 ~]# setenforce 0

[root@Web1 ~]# service httpd start