爬虫基础之HttpClient访问网站

HttpClient坐标

org.apache.httpcomponents

httpclient

4.5.3

GET方式：

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class HttpClientGet {
    public static void main(String[] args) throws IOException {
        //1.创建HttpClient对象
        CloseableHttpClient aDefault = HttpClients.createDefault();
        //2.指定请求方式
        HttpGet httpGet = new HttpGet("http://www.bingosoft.net");
        //3.可选的，封装请求参数
        //3.1封装请求体
        httpGet.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding","gzip, deflate");
        httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9");
        httpGet.setHeader("Cache-Control","max-age=0");
        httpGet.setHeader("Connection","keep-alive");
        httpGet.setHeader("Host","www.itcast.cn");
        httpGet.setHeader("Upgrade-Insecure-Requests","1");
        httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36");
        //4.发送请求
        CloseableHttpResponse execute = aDefault.execute(httpGet);
        //5.获取数据
        //5.1获取状态码
        if(execute.getStatusLine().getStatusCode()==200){
            String html = EntityUtils.toString(execute.getEntity(), "UTF-8");
            System.out.println(html);
        }
    }
}

POST方式：

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class HttpClientPost {

    public static void main(String[] args) throws IOException {
        //1.创建HttpClient对象
        CloseableHttpClient aDefault = HttpClients.createDefault();
        //2.指定请求
        HttpPost httpPost = new HttpPost("http://www.bingosoft.net");
        //3.封装请求参数
        List params = new ArrayList();
        params.add(new BasicNameValuePair("username","zs"));
        params.add(new BasicNameValuePair("password","123"));
        HttpEntity httpEntity = new UrlEncodedFormEntity(params);
        httpPost.setEntity(httpEntity);
        //3.1设置请求体
        httpPost.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        httpPost.setHeader("Accept-Encoding","gzip, deflate");
        httpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9");
        httpPost.setHeader("Cache-Control","max-age=0");
        httpPost.setHeader("Connection","keep-alive");
        httpPost.setHeader("Host","www.itcast.cn");
        httpPost.setHeader("Upgrade-Insecure-Requests","1");
        httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36");
        //4.执行请求
        CloseableHttpResponse response = aDefault.execute(httpPost);
        //5.获取数据
        //5.1获取响应头数据
        Header[] headers = response.getHeaders("Accept-Ranges");
        System.out.println(Arrays.toString(headers));
        if(response.getStatusLine().getStatusCode()==200){
            //5.2获取响应体数据
            String html = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(html);
        }
    }
}