HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置问题

 

先设置4个：



              



[csharp] 

webrequest.ServicePoint.Expect100Continue = false; 

               //是否使用 Nagle 不使用 提高效率 

               webrequest.ServicePoint.UseNagleAlgorithm = false; 

               //最大连接数 

               webrequest.ServicePoint.ConnectionLimit = 65500; 

               //数据是否缓冲 false 提高效率  

               webrequest.AllowWriteStreamBuffering = false; 



 



代理的使用：我用的代理都是不要账号和密码的，因为要不是免费的，要是是我们自己的，对来访IP 有限制



           



[csharp] 

if (proxy!=null) 

               { 

                   model.proxy = proxy; 

                   WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); 

                   myProxy.Credentials = new NetworkCredential("", ""); 

                   webrequest.Proxy = myProxy; 

               } 

               else 

               { 

                   webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); 

               } 

 



webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");



带上gzip，如果对方支持gzip，流量上可以节省五分之四，原来250KB，用gzip压缩有 大概在50KB，减少带宽压力，增加速度



值得注意的是，如果是gzip，返回值必须要gzip解压的。



 



[csharp] 

if (response.ContentEncoding.ToLower().Contains("gzip")) 

                   { 

                       using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) 

                       { 

                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 

                           { 

 

                               model.html = reader.ReadToEnd(); 

                           } 

                       } 

                   } 

 



还有一点：gzip解压的时候回耗费CPU，所以这个需要注意下



 



附完成方法



 



 



[csharp]

/// <summary> 

       /// Get 方式 获取数据 

       /// </summary> 

       /// <param name="webUrl"></param> 

       /// <returns></returns> 

       public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy) 

       { 

           Model.WebRequestReturnModel model = new Model.WebRequestReturnModel(); 

           HttpWebRequest webrequest = null; 

           try 

           { 

               webrequest = WebRequest.Create(webUrl) as HttpWebRequest; 

                

               webrequest.ServicePoint.Expect100Continue = false; 

                webrequest.ServicePoint.UseNagleAlgorithm = false; 

                webrequest.ServicePoint.ConnectionLimit = 65500; 

               webrequest.AllowWriteStreamBuffering = false; 

               if (proxy!=null) 

               { 

                   model.proxy = proxy; 

                   WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); 

                   myProxy.Credentials = new NetworkCredential("", ""); 

                   webrequest.Proxy = myProxy; 

               } 

               else 

               { 

                   webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); 

               } 

               //设置其他的herader 

               webrequest =WebRequestSetHeaders(webrequest); 

               if (!String.IsNullOrEmpty(refer)) 

               { 

                   webrequest.Referer = refer; 

               } 

               else 

               { 

                   webrequest.Referer = webUrl; 

               } 

               webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate"); 

               webrequest.CookieContainer = cookieContainer; 

               webrequest.Timeout = 5000; 

               webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"; 

               webrequest.Accept = "*/*"; 

               webrequest.KeepAlive = true; 

               webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); 

 

               using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse()) 

               { 

 

                   if (response.ContentEncoding.ToLower().Contains("gzip")) 

                   { 

                       using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) 

                       { 

                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 

                           { 

 

                               model.html = reader.ReadToEnd(); 

                           } 

                       } 

                   } 

                   else if (response.ContentEncoding.ToLower().Contains("deflate")) 

                   { 

                       using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress)) 

                       { 

                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 

                           { 

 

                               model.html = reader.ReadToEnd(); 

                           } 

 

                       } 

                   } 

                   else 

                   { 

                       using (Stream stream = response.GetResponseStream()) 

                       { 

                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 

                           { 

 

                               model.html = reader.ReadToEnd(); 

                           } 

                       } 

                   } 

               } 

           } 

           catch (Exception ex) 

           { 

               model.exception = ex; 

               model.html = string.Empty; 

 

           } 

           model.cookieContainer = cookieContainer; 

           return model; 

       }