HttpClient解析页面总结

1、遇到安全控件证书（单向和双向认证）

//安全证书的处理
        Protocol myhttps = new Protocol("https",
                new MySecureProtocolSocketFactory(), 443);
        Protocol.registerProtocol("https", myhttps);

 

package com.bwzy.jiaotongyunguan.util;

import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.UnknownHostException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;

import javax.net.SocketFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.params.HttpConnectionParams;
import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;

public class MySecureProtocolSocketFactory implements SecureProtocolSocketFactory {
    static{
        System.out.println(">>>>in MySecureProtocolSocketFactory>>");
    }
    private SSLContext sslcontext = null;

    private SSLContext createSSLContext() {
        SSLContext sslcontext=null;
        try {
            sslcontext = SSLContext.getInstance("SSL");
            sslcontext.init(null, new TrustManager[]{new TrustAnyTrustManager()}, new java.security.SecureRandom());
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        }
        return sslcontext;
    }

    private SSLContext getSSLContext() {
        if (this.sslcontext == null) {
            this.sslcontext = createSSLContext();
        }
        return this.sslcontext;
    }

    public Socket createSocket(Socket socket, String host, int port, boolean autoClose)
            throws IOException, UnknownHostException {
        return getSSLContext().getSocketFactory().createSocket(
                socket,
                host,
                port,
                autoClose
            );
    }

    public Socket createSocket(String host, int port) throws IOException,
            UnknownHostException {
        return getSSLContext().getSocketFactory().createSocket(
                host,
                port
            );
    }


    public Socket createSocket(String host, int port, InetAddress clientHost, int clientPort)
            throws IOException, UnknownHostException {
        return getSSLContext().getSocketFactory().createSocket(host, port, clientHost, clientPort);
    }

    public Socket createSocket(String host, int port, InetAddress localAddress,
            int localPort, HttpConnectionParams params) throws IOException,
            UnknownHostException, ConnectTimeoutException {
        if (params == null) {
            throw new IllegalArgumentException("Parameters may not be null");
        }
        int timeout = params.getConnectionTimeout();
        SocketFactory socketfactory = getSSLContext().getSocketFactory();
        if (timeout == 0) {
            return socketfactory.createSocket(host, port, localAddress, localPort);
        } else {
            Socket socket = socketfactory.createSocket();
            SocketAddress localaddr = new InetSocketAddress(localAddress, localPort);
            SocketAddress remoteaddr = new InetSocketAddress(host, port);
            socket.bind(localaddr);
            socket.connect(remoteaddr, timeout);
            return socket;
        }
    }

    //自定义私有类
    private static class TrustAnyTrustManager implements X509TrustManager {

        public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
        }

        public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
        }

        public X509Certificate[] getAcceptedIssuers() {
            return new X509Certificate[]{};
        }
    }


}

所需要的jar包有commons-httpclient-3.1.jar、 commons-codec-1.7.jar 、httpcore.jar、commons-logging.jar

HttpClient相当于一个浏览器（但不支持javascript脚本）

//创建一个HttpClient

HttpClient client = new HttpClient();

//已某种方式发出请求（7中请求方式包括：Get、post、head……），接下来以Get和post来说明问题。

//创建一个请求方式

GetMethod get = new GetMethod(url);

//其中Url是访问的网址（解析的目标）,如果你解析百度首页String url="http://www.baidu.com"

//通过client的该方法来发出请求，参数是请求的方式。

client.executeMethode(get);

//执行之后调用method的方法来获取client的响应。

get.getResponseBodyAsString();

//这样就可以得到百度页面的HTML代码，你可以从代码中获取你所需要的数据。

//如果你的请求指向的是一个文件，如word文档。你可以以流的方式获取。

get.getResponseBodyAsStream();

//请求方式有一些方法，能获取和修改请求头和相应头的信息，还可以设置一些编码方式。接下来是一段获取流的代码。

get.setURI(new URI(url, false, "gb2312"));
            httpClient.executeMethod(get);
            InputStream in = get.getResponseBodyAsStream();
            ByteArrayOutputStream outStream = new ByteArrayOutputStream();
            int b = 0;
            while ((b = in.read()) != -1) {
                outStream.write(b);
            }
            byte[] bytes = outStream.toByteArray();
            long tempLong = System.currentTimeMillis();
            String[] paths = PdfConverterServlet.class.getClassLoader().getResource("/").getPath().split("/");
            String path ="";
            for (int i = 1; i < paths.length-2; i++) {
                path+=paths[i]+"/";
            }
            System.out.println(path);
            OutputStream out = new FileOutputStream(new File(path+"fileTemp/",
                    tempLong + ".doc"));
            out.write(bytes);
            out.close();

Post方式请求

大家知道，Get和Post在参数上有区别，Get方式的请求可以直接在链接地址后面跟上参数列表，而Post却不能。那么Post如何传参呢？

 

post = new PostMethod(url);
            NameValuePair[] param = {
                    new NameValuePair("actionflag", (String) params
                            .get("actionflag")),
                    new NameValuePair("userlogid", (String) params
                            .get("userlogid")),
                    new NameValuePair("isdeliverdo", (String) params
                            .get("isdeliverdo")),
                    new NameValuePair("editFlag", (String) params
                            .get("editFlag")),
                    new NameValuePair("isflow", (String) params.get("isflow")),
                    new NameValuePair("workid", (String) params.get("workid")),
                    new NameValuePair("flowlogid", (String) params
                            .get("flowlogid")),
                    new NameValuePair("jflowobjectno", (String) params
                            .get("jflowobjectno")),
                    new NameValuePair("actionfrom", (String) params
                            .get("actionfrom")),
                    new NameValuePair("scopeids", (String) params
                            .get("scopeids")),
                    new NameValuePair("biaoti", (String) params.get("biaoti")),
                    new NameValuePair("chaosong", (String) params
                            .get("chaosong")),
                    new NameValuePair("deptname", (String) params
                            .get("deptname")),
                    new NameValuePair("fengfariqicn", (String) params
                            .get("fengfariqicn")),
                    new NameValuePair("fenshu", (String) params.get("fenshu")),
                    new NameValuePair("miji", (String) params.get("miji")),
                    new NameValuePair("nigaobumen", (String) params
                            .get("nigaobumen")),
                    new NameValuePair("nigaobumenlogin", (String) params
                            .get("nigaobumenlogin")),
                    new NameValuePair("nigaobumenid", (String) params
                            .get("nigaobumenid")),
                    new NameValuePair("nigaoren", (String) params
                            .get("nigaoren")),
                    new NameValuePair("nigaoriqi", (String) params
                            .get("nigaoriqi")),
                    new NameValuePair("zhusong", URLDecoder
                            .decode((String) params.get("zhusong"))), };
            System.out.println();
            post.addParameters(param);
            httpClient.executeMethod(post);
            doc = Jsoup.parse(post.getResponseBodyAsString());