httpclient模拟需要登陆之后才能访问第三方网站(并且需要一些人工参与的操作)

httpclient模拟用户登陆并访问目标页面....作个个人记录！

最近接了个功能，调用第三方的API获取数据，入库到我们自己平台！但是提供方网站上的东西比较简陋：

简单说明下：

1.获取数据的接口需要唯一标识access_token；

2.access_token的获取需要通过code去获取；

3.code的获取需要引导用户(第三方平台的登陆用户)打开一个授权页面进行授权之后，重定向地址上 而假如没有登陆还需要你先进行登陆；

难点就在这个需要引导用户打开授权页面进行授权

1). 这里面我们直接获取数据，用户就是我们自己在该第三方平台的账号；

2).  没有登陆直接去访问会跳转到登陆页面；

3). 登陆了之后，会有个授权页面，需要手动去点击授权按钮才真正跳转；

如下列图：

然后查看上述页面的源代码，可找到发送请求的URL以及参数：

登陆的地址和参数：

点击授权连接按钮所作的操作：

上代码(HttpClients模拟 + Jsoup解析html)

-----此例是比较简单的没有验证码的登录页面

package com.fulihui.daweixinke.test.conn;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.near.toolkit.security.codec.MD5Coder;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static java.lang.System.out;

/**
 * 
 * 模拟需要登陆之后才能访问第三方网站
 * 并且需要一些人工参与的操作
 * 
 *
 * @author wang_wx
 * @Date 2017-08-24
 */
public class EasyLoginNextPageA {

    private String userName = "我是用户名";

    private String password = "我是密码";

    @Test
    public void loginNext() throws IOException {

        BasicCookieStore cookieStore = new BasicCookieStore();
        // 全局用这一个httpClient对象模拟真实的一个浏览器中操作
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();
        try {
            // 模拟用户登录
            HttpPost httpLogin = new HttpPost("xxxxxxxxxxxxxx/index.php/auth/auth/login");// 指向一个没有验证码的登录页面
            List nvps = new ArrayList<>();
            nvps.add(new BasicNameValuePair("username", userName));// 用户名对应的key
            nvps.add(new BasicNameValuePair("password", MD5Coder.md5Encode(password)));// 密码对应的key
            httpLogin.setEntity(new UrlEncodedFormEntity(nvps));
            CloseableHttpResponse respLogin = httpClient.execute(httpLogin);
            try {
                HttpEntity entity = respLogin.getEntity();
                out.println("respLogin------------>>" + respLogin.toString());
                out.println("Login form get: " + respLogin.getStatusLine());
                EntityUtils.consume(entity);
                out.println("Initial set of cookies:");
                List cookies = cookieStore.getCookies();
                if (cookies.isEmpty()) {
                    out.println("None");
                } else {
                    for (int i = 0; i < cookies.size(); i++) {
                        out.println("Cookie-" + i + "==" + cookies.get(i).toString());
                    }
                }
            } finally {
                respLogin.close();
            }

            // 利用会话保持，继续访问目标地址
            HttpGet httpGetAuth = new HttpGet("xxxxxxxxxxxxxxx/index.php/Auth/Auth/auth?app_id=xxxxxx&redirect_uri=回调地址&response_type=code");
            CloseableHttpResponse respAuth = httpClient.execute(httpGetAuth);
            String entityAuthStr = "";
            try {
                out.println(respAuth.getStatusLine());
                HttpEntity entityAuth = respAuth.getEntity();
                entityAuthStr = EntityUtils.toString(entityAuth);
                out.println("get Auth cookies:");
                List cookies = cookieStore.getCookies();
                if (cookies.isEmpty()) {
                    out.println("None");
                } else {
                    for (int i = 0; i < cookies.size(); i++) {
                        out.println("- " + cookies.get(i).toString());
                    }
                }
                out.println("访问目标地址的结果--------------------->>" + entityAuthStr);//把结果打印出来看一下
                EntityUtils.consume(entityAuth);
            } finally {
                respAuth.close();
            }
            // 解析登陆之后访问目标地址的html页面 获取目标form表单元素
            Document doc = Jsoup.parseBodyFragment(entityAuthStr);
            String title = doc.title();
            out.println("title-------->>" + title);
            Element element = doc.body();
            Elements form = element.select("[name='authForm']");// 授权的form表单
            out.println(form);
            String actionUrl = form.attr("action");
            String app_id = element.select("[name='app_id']").val();
            String redirect_uri = element.select("[name='redirect_uri']").val();
            String state = element.select("[name='state']").val();
            String key = element.select("[name='key']").val();
            String format = element.select("[name='format']").val();

            // 利用会话保持，继续模拟点击授权按钮 提交form表单
            HttpPost httpPostGetCode = new HttpPost(actionUrl);
            List nvps2 = new ArrayList<>();
            nvps2.add(new BasicNameValuePair("app_id", app_id));
            nvps2.add(new BasicNameValuePair("redirect_uri", redirect_uri));
            nvps2.add(new BasicNameValuePair("state", state));
            nvps2.add(new BasicNameValuePair("key", key));
            nvps2.add(new BasicNameValuePair("format", format));
            httpPostGetCode.setEntity(new UrlEncodedFormEntity(nvps2));
            CloseableHttpResponse respGetCode = httpClient.execute(httpPostGetCode);
            try {
                HttpEntity entityGetCode = respGetCode.getEntity();
                out.println("respAuth------------>>" + respGetCode.toString());
                // 最终目的 获取Location中的url中的某一值
                Header location = respGetCode.getFirstHeader("Location");
                out.println("location------------>>" + location.getValue());
                EntityUtils.consume(entityGetCode);
            } finally {
                respGetCode.close();
            }
        } finally {
            httpClient.close();
        }
    }

    @Test
    public void test111() {
        String password = "我是密码";
        String encode = MD5Coder.md5Encode(password);
        out.println("encode的值是：---" + encode + "，当前方法=EasyLoginNextPageA.test111()");
    }
}

打印日志：