【爬虫 | 4. HttpClient】

学习链接

4.1 Get 请求

package cn.itcast.crawler.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpGetTest {
    public static void main(String[] args) {
        // 创建HttpClient对象（打开浏览器）
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建HttpGet对象，设置url访问地址（输入网址）
        HttpGet httpGet = new HttpGet("https://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
            // 使用HttpClient发起请求，获取response（键入请求，返回响应）
            response = httpClient.execute(httpGet);

            // 解析响应（将响应转换成相应的信息）
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

4.2 带参数的 Get 请求

package cn.itcast.crawler.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpGetParamTest {
    public static void main(String[] args) throws Exception {
        // 创建HttpClient对象（打开浏览器）
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 设置请求地址是：https://www.bilibili.com/video/BV1cE411u7RA?p=5&spm_id_from=pageDriver
        // 创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("https://www.bilibili.com/video/BV1cE411u7RA");
        // 设置参数
        uriBuilder.setParameter("p", "5");
        uriBuilder.addParameter("spm_id_from", "pageDriver");

        // 创建HttpGet对象，设置url访问地址（输入网址）
        HttpGet httpGet = new HttpGet(uriBuilder.build());

        System.out.println("Request info: " + httpGet);

        CloseableHttpResponse response = null;
        try {
            // 使用HttpClient发起请求，获取response（键入请求，返回响应）
            response = httpClient.execute(httpGet);

            // 解析响应（将响应转换成相应的信息）
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

4.3 POST 请求

package cn.itcast.crawler.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpPostTest {
    public static void main(String[] args) {
        // 创建HttpClient对象（打开浏览器）
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建HttpPost对象，设置url访问地址（输入网址）
        HttpPost httpPost = new HttpPost("https://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
            // 使用HttpClient发起请求，获取response（键入请求，返回响应）
            response = httpClient.execute(httpPost);

            // 解析响应（将响应转换成相应的信息）
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

4.4 带参数的 POST 请求

package cn.itcast.crawler.test;

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class HttpPostParamTest {
    public static void main(String[] args) throws Exception {
        // 创建HttpClient对象（打开浏览器）
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建HttpPost对象，设置url访问地址（输入网址）
        HttpPost httpPost = new HttpPost("https://www.bilibili.com/video/BV1cE411u7RA");

        // 声明List集合，封装表单中的参数
        List<NameValuePair> params = new ArrayList<NameValuePair>();
        // 设置请求地址是：https://www.bilibili.com/video/BV1cE411u7RA?p=5&spm_id_from=pageDriver
        params.add(new BasicNameValuePair("p", "5"));
        params.add(new BasicNameValuePair("spm_id_from", "pageDriver"));

        // 创建表单的Entity对象，第一个参数是封装好的数据表单，第二个参数是编码
        UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "utf8");

        // 设置表单的Entity对象到Post请求中
        httpPost.setEntity(formEntity);

        CloseableHttpResponse response = null;
        try {
            // 使用HttpClient发起请求，获取response（键入请求，返回响应）
            response = httpClient.execute(httpPost);

            // 解析响应（将响应转换成相应的信息）
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

4.5 连接池

package cn.itcast.crawler.test;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpClientPoolTest {
    public static void main(String[] args) {
        // 创建连接池管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

        // 设置最大连接数
        cm.setMaxTotal(100);

        // 设置每个主机的最大连接数（可能同时爬取多个主机的数据，这里是分配连接数量）
        cm.setDefaultMaxPerRoute(10);

        // 使用连接池管理器发起请求
        doGet(cm);
        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        // 不是每次创建新的HttpClient，而是从连接池中获取HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();

        HttpGet httpGet= new HttpGet("https://www.itcast.cn");

        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);

            if(response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                // 不能关闭HttpClient，由连接池管理HttpClient
                // httpClient.close();
            }
        }
    }
}

4.6 请求参数

package cn.itcast.crawler.test;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpConfigTest {
    public static void main(String[] args) {
        // 创建HttpClient对象（打开浏览器）
        CloseableHttpClient httpClient = HttpClients.createDefault();

        // 创建HttpGet对象，设置url访问地址（输入网址）
        HttpGet httpGet = new HttpGet("https://www.itcast.cn");

        // 配置请求信息
        RequestConfig config = RequestConfig.custom().setConnectTimeout(1000) // 设置创建连接的最长时间，单位是毫秒
                .setConnectionRequestTimeout(500) // 设置获取连接的最长时间，单位是毫秒
                .setSocketTimeout(10*1000) // 设置数据传输的最长时间，单位是毫秒
                .build();

        // 给请求设置请求信息
        httpGet.setConfig(config);

        CloseableHttpResponse response = null;
        try {
            // 使用HttpClient发起请求，获取response（键入请求，返回响应）
            response = httpClient.execute(httpGet);

            // 解析响应（将响应转换成相应的信息）
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(), "utf8");
                System.out.println(content.length());
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            //关闭response
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                httpClient.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}