使用varnish来做静态文件缓存加速

 varnish这几年已经应用比较广泛且成熟了，与squid相比，它具有以下优点：

1、稳定性更好，在处理相同的负载情况下，squid发生故障的概率要高于varnish

2、访问速度更快，因为varnish的数据存储在内存中，而squid是存储在硬盘上。

3、varnish支持并发能力比squid要强，它释放tcp链接要比squid快，所以可以支持更多的链接

4、varnish在管理方面比squid强，他可以通过管理端口使用正则来清理部分缓存，squid是做不到的

5、varnish可以起多个进程，而squid是单cpu单核处理的

varnish的缺点：

1、重启服务后，内存数据清除，需要到后端服务器重新获取数据

安装一些系统需要的包

 

yum install gcc-c++
yum install gcc
yum install libtool
yum install ncurses-devel #如果这个不装，varnishstat不会安装，这个rhel5的一个bug

下载pcre和varnish

 

cd /opt
wget http://repo.varnish-cache.org/source/varnish-3.0.1.tar.gz
wget http://downloads.sourceforge.net/project/pcre/pcre/8.30/pcre-8.30.zip?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fpcre%2Ffiles%2Fpcre%2F&ts=1342770754&use_mirror=ncu

安装pcre

 

unzip pcre-8.30.zip
cd pcre-8.30
 ./configure --prefix=/usr/local/pcre
make && make install

安装varnish

 

export PKG_CONFIG_PATH=/usr/local/pcre/lib/pkgconfig
useradd  -s /sbin/nologin varnish
 mkdir -p /data/varnish/cache
mkdir -p /data/varnish/log
chown -R varnish:varnish /data/varnish
tar zxvf varnish-3.0.1.tar.gz
cd varnish-3.0.1
./configuare --prefix=/usr/local/varnish
make && make install

cp redhat/varnish.initrc /etc/init.d/varnish

cp redhat/varnish.sysconfig /etc/sysconfig/varnish

 

vim /etc/sysconfig/varnish

 

NFILES=131072
MEMLOCK=182000
DAEMON_OPTS="-a 0.0.0.0:80 \  #监听的地址和端口
             -T 127.0.0.1:3500 \  #管理地址和端口
             -f /usr/local/varnish/etc/vcl.conf\  #加载的配置
             -u varnish -g varnish \  #用户和组
             -w 10,51200,10 \  #min,max,timeout
             -n /data/varnish/cache \
             -s file,/data/varnish/cache/varnish_cache.data,4G" #-s 用来定义存储类型和存储容量。Mmap file是使用文件的形式存储，然后再一次性加载到内存中。这种方式如果在访问量很大的情况下，对IO有一定的压力
             #-s malloc,2G #这种方式是直接分配内存，不写文件。对IO无压力

chmod 775 /etc/init.d/varnish

vim /usr/local/varnish/etc/vcl.conf

 

#Cache for linuxtone sites
#backend vhost
backend webserver {
.host = "192.168.3.30"; #后端server IP
.port = "80";
}
#acl 运行purge缓存的3个IP
acl purge { 　
  "localhost";
  "127.0.0.1";
  "10.13.3.0"/24;
}
sub vcl_recv {
        if (req.http.Accept-Encoding) {
            if (req.url ~ "\.(jpg|png|gif|jpeg|flv)$" ) {
                remove req.http.Accept-Encoding;
                remove req.http.Cookie;
            } else if (req.http.Accept-Encoding ~ "gzip") {
                set req.http.Accept-Encoding = "gzip";
            } else if (req.http.Accept-Encoding ~ "deflate") {
                set req.http.Accept-Encoding = "deflate";
            } else {
                remove req.http.Accept-Encoding;
            }
        }
           if (req.http.host ~ "(.*)taobao.org") {
                       set req.backend = webserver;
                 }
            else {
                        error 404 "This website is maintaining or not exist";
                }
  if (req.request == "PURGE") {
     if (!client.ip ~purge) {
       error 405 "Not Allowed";
   }
#.dd.....
   return(lookup);
  }
#...GET...url...jpg,png,gif. ..cookie
  if (req.request == "GET"&& req.url ~ "\.(png|gif|jpeg|jpg|ico|swf|css|js|html|htm|gz|tgz|bz2|tbz|mp3|ogg|mp4|flv|f4v|pdf)$") {
        unset req.http.cookie;
  }
#..GET...url.php....cache....
  if (req.request =="GET"&&req.url ~ "\.php($|\?)"){
        return (pass);
  }
#     if (req.restarts == 0) {
        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
                req.http.X-Forwarded-For + ", " + client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
#   }
#........pipe..
    if (req.request != "GET" &&
      req.request != "HEAD" &&
      req.request != "PUT" &&
      req.request != "POST" &&
      req.request != "TRACE" &&
      req.request != "OPTIONS" &&
      req.request != "DELETE") {
        return (pipe);
    }
#..GET .HEAD.....
    if (req.request != "GET" && req.request != "HEAD") {
        return (pass);
    }
    if (req.http.Authorization) {
        return (pass);
    }
    return (lookup);
}
#..url+host hash......
sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (hash);
}
# .....purge .....
sub vcl_hit {
   if (req.request == "PURGE") {
       set obj.ttl = 0s;
       error 200 "Purged";
    }
    return (deliver);
}
sub vcl_fetch {
          if (req.url ~ "\.(jpeg|jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|ico|swf|flv|dmg|js|css|html|htm)$") {
                   set beresp.ttl = 2d;
                   set beresp.http.expires = beresp.ttl;
                   set beresp.http.Cache-Control = "max-age=172800";
                   unset beresp.http.set-cookie;
          }
          if (req.url ~ "\.(dmg|js|css|html|htm)$") {
                   set beresp.do_gzip = true;
          }
          if (beresp.status == 503) {
                         set beresp.saintmode = 15s;
          }
}
sub vcl_deliver {
        set resp.http.x-hits = obj.hits ;
        if (obj.hits > 0) {
                set resp.http.X-Cache = "HIT You!";
        } else {
                set resp.http.X-Cache = "MISS Me!";
        }
} 

/etc/init.d/varnish start #启动varnish

启动成功后，进程中会出现两个varnishd进程。一个master主进程和一个worker工作子进程。master进程读取存储配置命令，进行初始化。子进程则分配线程进行cache工作。

/usr/local/varnish/bin/varnishlog -n /data/varnish/cache

/usr/local/varnish/bin/varnishncsa  -n  /data/varnish/cache

#多次执行该语句，可以测试是否命中缓存

curl -I http://wfstatic.taobao.org/Resources/images/bg_welcome.png

#列出最近被清除的内容

/usr/local/varnish/bin/varnishadm -T 127.0.0.1:3500 ban.list 

#清除指定的文件，后面可以用正则来匹配

/usr/local/varnish/bin/varnishadm -T 127.0.0.1:3500 ban.url /resources/mvcres/style/default/form.css 

#之前的版本，清除缓存使用的是purge.url