【四】C# 基于WPF 使用CefSharp来做万能爬虫---扩展 一个httpClient访问辅助类可设置代理
目录
【一】C# 基于WPF 使用CefSharp来做万能爬虫
【二】C# 基于WPF 使用CefSharp来做万能爬虫
【三】C# 基于WPF 使用CefSharp来做万能爬虫
【四】C# 基于WPF 使用CefSharp来做万能爬虫---扩展 一个httpClient访问辅助类可设置代理
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Security;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
namespace HttpWebHelper
{
///
/// 请求URL的一个辅助类
///
public static class HttpRequest
{
///
/// 请求的封装，封装了默认请求方式
///
///
///
///
///
/// query = null, object postdata = null)
{
Args httpArgs = new Args();
httpArgs.Url = url;
httpArgs.Method = method;
httpArgs.cookStr = cookStr;
if (query != null)
httpArgs.QueryDic = query;
if (postdata != null)
httpArgs.postData.Data = postdata;
return Request(httpArgs);
}
///
/// http请求的集合体
///
/// 请求类
///
/// 处理返回值
///
///
///
/// 创建请求HttpClient
///
///
///
/// 处理返回的htmlSource
///
///
/// 处理返回值里面的Header
///
///
/// HandleHeaders(WebHeaderCollection header)
{
Dictionary valuePairs = new Dictionary();
foreach (var key in header.AllKeys)
{
string values = string.Empty;
values = header.Get(key);
valuePairs.Add(key, values);
}
return valuePairs;
}
///
/// 保存成文件
///
///
///
/// list = new List();
while (true)
{
int data = sream.ReadByte();
if (data == -1)
break;
list.Add((byte)data);
}
File.WriteAllBytes(savePath.FullName, list.ToArray());
}
return true;
}
catch { return false; }
}
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
///
/// 获取该网页的源码 //兼容gzip格式的
///
/// 该网页的HttpWebResponse
///
/// MD5摘要算法
///
///
///
/// http返回值内容的格式化
///
public class ReturnMessage
{
///
/// 请求回来的html详细内容
///
public string HtmlSource { get; set; } = string.Empty;
///
/// 请求回来的详细内容如果是Json的话会自动转化为js对象
///
public JToken JsonObject { get; set; } = null;
///
/// 请求返回回来的状态码
///
public int Status { get; set; } = 200;
///
/// 请求返回回来的所有cookie
///
public string CookieStr { get; set; } = string.Empty;
///
/// 请求的Headers
///
public Dictionary Hearders { get; set; } = null;
}
///
/// http请求时的参数格式化
///
public class Args
{
///
/// 请求的url
///
public string Url { get; set; } = string.Empty;
///
/// 请求的query参数
///
public Dictionary QueryDic { get; set; } = new Dictionary();
///
/// http请求类型
///
public MethodEnum Method { get; set; } = MethodEnum.Get;
///
/// http请求postStr
///
public PostData postData { get; set; } = new PostData();
///
/// http请求时的hearder
///
public Dictionary Hearders { get; set; } = null;
///
/// http请求时的cookie
///
public string cookStr { get; set; } = string.Empty;
///
/// 代理
///
public Poxy poxy { get; set; } = new Poxy();
public int TimeOut { get; set; } = 20;
///
/// 用户身份信息头
///
public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36";
///
/// 表单方式
///
public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
///
/// 链接是否保持存活
///
public bool KeepAlive { get; set; } = false;
///
/// 如果请求应自动遵循来自 Internet 资源的重定向响应，则为 true；否则为 false。 默认值为 false。
///
public bool AllowAutoRedirect { get; set; } = false;
///
/// 是否是文件
///
public bool isFile { get; set; } = false;
///
/// 文件存储的路径
///
public FileInfo FileInfo { get; set; } = null;
}
///
/// 代理配置
///
public class Poxy
{
///
/// 代理服务器IP
///
public string Ip { get; set; } = string.Empty;
///
/// 代理服务器端口号
///
public int Port { get; set; } = 80;
}
///
/// 提交数据
///
public class PostData
{
///
/// 请求body的方式 默认json
///
public ContentType contentType { get; set; } = ContentType.Json;
///
/// post提交的data的内容
///
public object Data { get; set; } = new object();
///
/// 编码格式
///
public Encoding encoding { get; set; } = Encoding.UTF8;
}
///
/// postData内容的格式
///
public enum ContentType
{
FromData,//表单提交
Json //json方式提交
}
///
/// 请求的类型
///
public enum MethodEnum
{
Get,
Post,
Put,
Delete
}
}
使用示例：
string url = "http://192.168.1.97:8001/api/NEWICM/NewCompanyModule/NewCompanyDetailInfo/GetContactInfo";
string cookStr="sessionStr=1234567";
var relut = HttpRequest.Request(url, MethodEnum.Get, cookStr);