使用代理IP网络爬虫的三种方式

import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketAddress;
import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SocketChannel;
import java.nio.charset.Charset;
import java.util.Scanner;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSession;

public class Tester {
    public static void main(String[] args) {
        String uri = "https://fpcy.hb-n-tax.gov.cn:443/WebQuery/yzmQuery";
        //String uri = "http://pv.sohu.com/cityjson?ie=utf-8";

        String host = "52.183.30.241";
        int port = 8888;

        try {
            test1(uri, host, port);
            //test2(uri, host, port);
            //test3(uri, host, port);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    static void test1(String uri, String host, Integer port) throws Exception {
        SocketAddress addr = new InetSocketAddress(host, port);
        Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
        IngoreSSL();//忽略HTTPS请求的SSL证书,按需要设置
        URLConnection connection = new URL(uri).openConnection(proxy);
        show(connection.getInputStream());
    }

    /**
     * 该方法 如果代理ip失败，会使用本地ip
     * @param uri
     * @param host
     * @param port
     * @throws Exception
     */
    static void test2(String uri, String host, Integer port) throws Exception {
        System.setProperty("http.proxySet", "true");
        System.setProperty("http.proxyHost", host);
        System.setProperty("http.proxyPort", port + "");
        IngoreSSL();
        URLConnection connection = new URL(uri).openConnection();
        show(connection.getInputStream());
    }

    static void test3(String uri, String host, Integer port) throws IOException {
        SocketChannel sc = SocketChannel.open();
        // 要设置连接超时
        sc.socket().connect(new InetSocketAddress(host, port), 3000);
        // 设置读超时
        sc.socket().setSoTimeout(3000);

        sc.write(Charset.forName("utf8").encode("GET " + uri + " \r\n\r\n"));
        ByteBuffer buffer = ByteBuffer.allocate(1024);
        InputStream is = sc.socket().getInputStream();
        ReadableByteChannel readCh = Channels.newChannel(is);
        while (readCh.read(buffer) != -1) {
            buffer.flip();
            System.out.println(Charset.forName("utf8").decode(buffer));
            buffer.clear();
        }
        sc.close();
    }

    static void show(InputStream in) throws IOException {
        Scanner cin = new Scanner(in);
        StringBuilder builder = new StringBuilder();
        while (cin.hasNext()) {
            builder.append(cin.nextLine());
        }
        cin.close();
        System.out.println(builder.toString());
    }

    /**
     * 忽略HTTPS请求的SSL证书，必须在openConnection之前调用
     * @throws Exception
     */
    static void IngoreSSL() throws Exception {
        trustAllHttpsCertificates();
        HostnameVerifier hv = new HostnameVerifier() {
            public boolean verify(String urlHostName, SSLSession session) {
                return true;
            }
        };
        HttpsURLConnection.setDefaultHostnameVerifier(hv);
    }

    private static void trustAllHttpsCertificates() throws Exception {
        javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[1];
        javax.net.ssl.TrustManager tm = new miTM();
        trustAllCerts[0] = tm;
        javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance("SSL");
        sc.init(null, trustAllCerts, null);
        javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
    }

    static class miTM implements javax.net.ssl.TrustManager, javax.net.ssl.X509TrustManager {
        public java.security.cert.X509Certificate[] getAcceptedIssuers() {
            return null;
        }

        public boolean isServerTrusted(java.security.cert.X509Certificate[] certs) {
            return true;
        }

        public boolean isClientTrusted(java.security.cert.X509Certificate[] certs) {
            return true;
        }

        public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException {
            return;
        }

        public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException {
            return;
        }
    }
}