持多核smp squid3.2 缓存反向代理【ok】~

squid终于支持多核心了~~~~~~~~~~~

 

从squid官方网站 http://www.squid-cache.org/Versions/下载  
稳定版有2.7和3.1供选择 测试版3.2已经加入了SMP支持，可以最大利用处理器资源，3.1之前的所有版本都是不支持SMP和多CPU。

 

  

yum -y install ntp make openssl openssl-devel pcre pcre-devel libpng libpng-devel libjpeg-6b libjpeg-devel-6b freetype freetype-devel gd gd-devel zlib zlib-devel gcc gcc-c++ libXpm libXpm-devel ncurses ncurses-devel libmcrypt libmcrypt-devel libxml2 libxml2-devel imake autoconf automake screen sysstat compat-libstdc++-33 curl curl-devel
wget http://www.squid-cache.org/Versions/v3/3.2/squid-3.2.0.18.tar.gz
tar -zvxf squid-3.2.0.18.tar.gz
cd squid-3.2.0.18
./configure --prefix=/usr/local/squid --enable-gnuregex --enable-async-io=80 --enable-cache-digests --enable-err-language="zh-cn" --enable-default-err-language="zh-cn" --enable-epoll --disable-internal-dns  --with-filedescriptors=20480 --enable-kill-parent-hack
make
make install

#创建squid相关目录

groupadd squid   #创建squid  squid用户组
useradd -g squid -s /sbin/nologin  #创建squid用户，并加入到squid组里，不允许登录系统

chown -R squid /usr/local/squid/   #修改squid的安装目录所属用户为squid用户
mkdir -p /var/cache1     #创建squid的第一个缓存目录
mkdir -p /var/cache2    #创建squid的第二个缓存目录
chown squid.squid -R /var/cache1 /var/squid2  #设置目录所有者

chmod -R 777 /var/cache1 /var/squid    #设置目录权限

#配置squid

将squid的初始配置文件备份，我们要重新创建配置文件

mv /usr/local/squid/etc/squid.conf /usr/local/squid/etc/squid.conf.bak

vi /usr/local/squid/etc/squid.conf

 

 

max_filedescriptors 65535
visible_hostname rui

acl SSL_ports port 443
acl Safe_ports port 80
acl CONNECT method CONNECT
acl myip src localhost
#http_access deny OverConnLimit

#acl web1 src 117.21.227.134
#acl web2 src .....
http_access allow manager localhost
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports


acl OverConnLimit maxconn 300
http_access deny OverConnLimit

http_access allow all
#header_access X-Forwarded-For allow all
#acl_uses_indirect_client  on
#follow_x_forwarded_for allow all
allow_underscore on
half_closed_clients off

cache_peer 222.174.95.21 parent 9011 0 no-query originserver name=web21
cache_peer 122.226.213.41 parent 80 0 no-query originserver name=web41

http_port 80 accel vhost vport

cache_peer_domain  web21  .upbar.net mypig.upbar.net .mypig.net .8goo.com .vitas.com.cn .ibar.cc  .360loss.com .99zyz.com
cache_peer_domain  web21  .grow100.com.cn .jpstore.net .jk0769.com .maxmancapsule.com .maxmancapsule.com.cn .maxmancapsules.com.tw .72sun.com .30jf.com .356jf.com .24jf.com .aidashan.com .she36.com
cache_peer_domain  web21  .grow100.net .gao36.com .gao36.net .way200.com .easy900.com .show1000.com .gft400.com .topgouwu.net
cache_peer_domain  web21  .igaofei.com .igaofei.net .mek123.com .mek123.net .yugutoo.com yututoo.com
cache_peer_domain  web41  .bibe.cn .yein.cc .youxia.cn

cache_peer_access web21 allow all
cache_peer_access web41 allow all

forwarded_for on

acl QUERY urlpath_regex cgi-bin   .cgi .php .avi .wmv .rm .ram .mpg .mpeg .zip .exe .asp .aspx
cache deny QUERY

reload_into_ims on

refresh_pattern ^ftp:           1440    20%     10080
refresh_pattern ^gopher:        1440    0%      1440
refresh_pattern .               0       20%     4320
refresh_pattern -i \.html 1440 100% 129600 reload-into-ims
refresh_pattern -i \.shtml 1440 100% 129600 reload-into-ims
refresh_pattern -i \.htm 1440 100% 129600 reload-into-ims
refresh_pattern -i \.gif 1440 100% 129600 ignore-reload
refresh_pattern -i \.jpg 1440 100% 129600 ignore-reload
refresh_pattern -i \.png 1440 100% 129600 reload-into-ims
refresh_pattern -i \.bmp 1440 100% 129600 reload-into-ims
refresh_pattern -i \.swf 1440 100% 129600 reload-into-ims
refresh_pattern -i \.flv 129600 100% 129600 ignore-reload
refresh_pattern -i \.js 1440 100% 129600 reload-into-ims
refresh_pattern -i \.css 1440 100% 129600 reload-into-ims


pid_filename /usr/local/squid/var/logs/squid.pid

#logformat squid  %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
#cache_log /usr/local/squid/var/logs/cache.log
#access_log /usr/local/squid/var/logs/access.log
access_log /dev/null
cache_log /dev/null
cache_store_log none

#error_directory none

cache_mem 256 MB
memory_pools_limit 312 MB
maximum_object_size_in_memory 4096 KB
max_open_disk_fds 0
minimum_object_size 0 KB
maximum_object_size 4096 KB
maximum_object_size_in_memory 4096 KB
memory_replacement_policy lru
cache_dir ufs /var/cache1 1024 16 256
#cache_dir null /tmp


EOF

 

 

maximum_object_size 是 能cache最大的文件大小。对应wmv，rm文件，建议设置为32768 kB maximum_object_size_in_memory 是在内存中cache的最大文件大小。 cache_mem 是SQUID可用到的最大内存。经实践，4G内存的服务器用2G；超过2G导致SQUID运行不稳 首先要分析SQUID所cache内容： 运行 squidclient -p 80 cache_object://localhost/info 能看到如下内容： Storage Swap size: 7549104 KB Storage Mem size: 418804 KB Mean Object Size: 160.46 KB Mean Object Size是平均内容大小，一般要把maximum_object_size_in_memory设置成离它最近的128的倍数。在这个例子中maximum_object_size_in_memory 的值应该是256kB。 cache_mem 一般设置成服务器内存的一半或更多，只要运行过程中LINUX没有使用SWAP就可以。 再就是按业务分SQUID。 比如某个论坛，用户能上载图片和视频；当然我们要把上载的图片、视频放在单独的域名上，比如img.example.com, video.example.com；这两个域名只提供静态文件服务。 根据统计，图片的平均大小在100KB，视频的平均大小在4M，差别是很大，应该建两个squid分别作图片和视频的CACHE。图片SQUID的 maximum_object_size_in_memory 设置为256KB，视频的SQUID的maximum_object_size_in_memory设置为8196KB。 Squid优化(2) Hot!探讨动态内容的CACHE。 BBS，论坛是典型动态内容，要保证内容更新及时的同时，提高访问速度，降低数据库负担不是个简单任务。经实践发现如下办法取得很好效果： 1) 配置SQUID，对动态内容强制CACHE，用到的配置参数是refresh_pattern refresh_pattern ^/forum/viewthread.php 1440 1000% 1440 ignore-reload /forum/viewthread.php的内容将强制保持1天 2) 修改论坛程序在用户回复帖子后，向SQUID发送PURGE命令清除相应帖子的页面CACHE，保证失效性 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~实现过这一功能,但是有时候生效,有时候无效,还未进一步查明原因.(Edit by Sean) 3) 有些频繁更新的页面可以不CACHE，用no_cache参数 acl no_forum_cache urlpath_regex ^/forum/forumdisplay.php no_cache DENY no_forum_cache

 

启动squid

检查 squid 配置文件正确与否

/usr/local/squid/sbin/squid -Nk parse

 

生成缓存目录

/usr/local/squid/sbin/squid -N –z

测试squid

/usr/local/squid/sbin/squid -N -d1

启动squid

/usr/local/squid/sbin/squid

ps -ef |grep squid     # 检查squid是否正常启动

 

配置squid开机启动

vi /etc/rc.d/rc.local

在最后添加

/usr/local/squid/sbin/squid

5.滚动日志文件

squid将大量的重要信息写入日志，假如写不进去了，squid会发生错误并退出。为了合理控制磁盘空间消耗，必须周期性的滚动日志文件。

/usr/local/squid/sbin/squid -k rotate

ls -alh /usr/local/squid/var/logs/

 

通过上图，可以看到日志已经被分割了，将此命令加入到crontab里，让它在每天的凌晨4点运行。

crontab -e

在最后添加

0 4 * * * /usr/local/squid/sbin/squid -k rotate

 

结束squid进程 /squid/sbin/squid –k shutdown
取得squid运行状态信息：/squid/bin/squidclient -p 80 mgr:info
取得squid内存使用情况：/squid/bin/squidclient -p 80 mgr:mem
取得squid已经缓存的列表：/squid/bin/squidclient -p 80 mgrbjects. use it carefully,it may crash
取得squid的磁盘使用情况：/squid/bin/squidclient -p 80 mgr:diskd
强制更新某个url：/squid/bin/squidclient -p 80 -m PURGE http://www.rui.com/
查命中率：
/soft/squid/bin/squidclient -h111.222.111.111 -p80 mgr:info
/soft/squid/bin/squidclient -h具体的IP -p80 mgr:info

 

 清理缓存~~~

qingli.sh      www.xiuxiukan.com

 

qingli.sh      jpg

qingli.sh       xiuxiukan.com 123  bbb  jpg

#!/bin/sh
squidcache_path="/squidcache"
squidclient_path="/home/local/squid/bin/squidclient"
#grep -a -r $1 $squidcache_path/* | grep "http:" | awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' > cache.txt
if [[ "$1" == "swf" || "$1" == "png" || "$1" == "jpg" || "$1" == "ico" || "$1" == "gif" || "$1" == "css" || "$1" == "js" || "$1" == "html" || "$1" == "shtml" || "$1" == "htm"   ]]; then
grep -a -r .$1 $squidcache_path/* | strings | grep "http:" | awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' | grep "$1$" | uniq > cache.txt
else
grep -a -r $1 $squidcache_path/* | strings | grep "http:" |grep $2$ |grep $3$|grep $4$|grep $5$ |grep $6$| awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' | uniq > cache.txt
fi
sed -i "s/\";$//g" cache.txt
cat cache.txt | while read LINE
do
$squidclient_path -p 80 -m PURGE $LINE
done

 

squid启动文件

cat >> /etc/init.d/squid <<EOF
#!/bin/bash
#chkconfig: 345 85 15
#description: squid test
#BY liyaoyi 2012-05-04

. /etc/rc.d/init.d/functions
squid="/usr/local/squid/sbin/squid"
prog="squid"
RETVAL=0
start() {
        echo -n $"Starting $prog: "
        daemon $squid -s
        RETVAL=$?
        echo
        return $RETVAL
        }
stop () {
        echo -n $"Stoping $prog: "
        daemon $squid -k shutdown
        echo
        return $RETVAL
        }
reload () {
        echo -n $"Reloading $prog: "
        daemon $squid -k reconfigure
        echo
        return $RETVAL
        }

case "$1" in
        start)
                start
                ;;
        stop)
                stop
                ;;
        reload)
                reload
                ;;
        restart)
                stop
                start
                ;;
        *)
        echo $"Usage: $0 {start|stop|restart|reload}"
                RETVAL=1
esac
exit $RETVAL
EOF

chmod a+x /etc/init.d/squid
chkconfig --add squid
chkconfig squid on
/etc/init.d/squid start