Java模拟HTTP的Get和Post请求(增强) Apache-httpclient

http://hc.apache.org/downloads.cgi

 

下面是代码：

import org.apache.commons.httpclient.HttpClient; 
import org.apache.commons.httpclient.HttpMethod; 
import org.apache.commons.httpclient.HttpStatus; 
import org.apache.commons.httpclient.URIException; 
import org.apache.commons.httpclient.methods.GetMethod; 
import org.apache.commons.httpclient.methods.PostMethod; 
import org.apache.commons.httpclient.params.HttpMethodParams; 
import org.apache.commons.httpclient.util.URIUtil; 
import org.apache.commons.lang.StringUtils; 
import org.apache.commons.logging.Log; 
import org.apache.commons.logging.LogFactory; 

import java.io.BufferedReader; 
import java.io.IOException; 
import java.io.InputStreamReader; 
import java.util.Map; 

/** 
* HTTP工具箱 
* 
* @author leizhimin 2009-6-19 16:36:18 
*/ 
public  final  class HttpTookit { 
         private  static Log log = LogFactory.getLog(HttpTookit. class); 

         /** 
         * 执行一个HTTP GET请求，返回请求响应的HTML 
         * 
         * @param url                 请求的URL地址 
         * @param queryString 请求的查询参数,可以为null 
         * @param charset         字符集 
         * @param pretty            是否美化 
         * @return 返回请求响应的HTML 
         */ 
         public  static String doGet(String url, String queryString, String charset,  boolean pretty) { 
                StringBuffer response =  new StringBuffer(); 
                HttpClient client =  new HttpClient(); 
                HttpMethod method =  new GetMethod(url); 
                 try { 
                         if (StringUtils.isNotBlank(queryString)) 
                                 //对get请求参数做了http请求默认编码，好像没有任何问题，汉字编码后，就成为%式样的字符串 
                                method.setQueryString(URIUtil.encodeQuery(queryString)); 
                        client.executeMethod(method); 
                         if (method.getStatusCode() == HttpStatus.SC_OK) { 
                                BufferedReader reader =  new BufferedReader( new InputStreamReader(method.getResponseBodyAsStream(), charset)); 
                                String line; 
                                 while ((line = reader.readLine()) !=  null) { 
                                         if (pretty) 
                                                response.append(line).append(System.getProperty( "line.separator"));
                                         else 
                                                response.append(line); 
                                } 
                                reader.close(); 
                        } 
                }  catch (URIException e) { 
                        log.error( "执行HTTP Get请求时，编码查询字符串“" + queryString +  "”发生异常！", e); 
                }  catch (IOException e) { 
                        log.error( "执行HTTP Get请求" + url +  "时，发生异常！", e); 
                }  finally { 
                        method.releaseConnection(); 
                } 
                 return response.toString(); 
        } 

         /** 
         * 执行一个HTTP POST请求，返回请求响应的HTML 
         * 
         * @param url         请求的URL地址 
         * @param params    请求的查询参数,可以为null 
         * @param charset 字符集 
         * @param pretty    是否美化 
         * @return 返回请求响应的HTML 
         */ 
         public  static String doPost(String url, Map<String, String> params, String charset,  boolean pretty) { 
                StringBuffer response =  new StringBuffer(); 
                HttpClient client =  new HttpClient(); 
                HttpMethod method =  new PostMethod(url); 
                 //设置Http Post数据 
                 if (params !=  null) { 
                        HttpMethodParams p =  new HttpMethodParams(); 
                         for (Map.Entry<String, String> entry : params.entrySet()) { 
                                p.setParameter(entry.getKey(), entry.getValue()); 
                        } 
                        method.setParams(p); 
                } 
                 try { 
                        client.executeMethod(method); 
                         if (method.getStatusCode() == HttpStatus.SC_OK) { 
                                BufferedReader reader =  new BufferedReader( new InputStreamReader(method.getResponseBodyAsStream(), charset)); 
                                String line; 
                                 while ((line = reader.readLine()) !=  null) { 
                                         if (pretty) 
                                                response.append(line).append(System.getProperty( "line.separator"));
                                         else 
                                                response.append(line); 
                                } 
                                reader.close(); 
                        } 
                }  catch (IOException e) { 
                        log.error( "执行HTTP Post请求" + url +  "时，发生异常！", e); 
                }  finally { 
                        method.releaseConnection(); 
                } 
                 return response.toString(); 
        } 

         public  static  void main(String[] args) { 
                String y = doGet( "http://video.sina.com.cn/life/tips.html", null, "GBK", true); 
                System.out.println(y); 
        } 
}

 

其实，这里的美化加入系统平台无关的换行符，可以任何平台上正确显示换行。

 

另外发现，通过JDK自带类库实现的请求，往往在请求一些url时，无法获取到请求结果。但此工具总能保证只要你的浏览器能打开，它总能忠实的将响应的内容反馈回来。

 

另外，将查看Java平台环境的代码给出来，很常用！

         public  static  void main(String[] args) { 
                Properties p = System.getProperties(); 
                 for (Map.Entry<Object, Object> entry : p.entrySet()) { 
                        System.out.println(entry.getKey() +  " = " + entry.getValue()); 
                } 
        }

 

java.runtime.name = Java(TM) SE Runtime Environment 
sun.boot.library.path = Q:\jdk6\jre\bin 
java.vm.version = 14.0-b16 
java.vm.vendor = Sun Microsystems Inc. 
java.vendor.url = http: //java.sun.com/ 
path.separator = ; 
idea.launcher.port = 7540 
java.vm.name = Java HotSpot(TM) Client VM 
file.encoding.pkg = sun.io 
sun.java.launcher = SUN_STANDARD 
user.country = CN 
sun.os.patch.level = Service Pack 3 
java.vm.specification.name = Java Virtual Machine Specification 
user.dir = E:\NVMS\Code\sinamovie 
java.runtime.version = 1.6.0_14-b08 
java.awt.graphicsenv = sun.awt.Win32GraphicsEnvironment 
java.endorsed.dirs = Q:\jdk6\jre\lib\endorsed 
os.arch = x86 
java.io.tmpdir = C:\DOCUME~1\ADMINI~1\LOCALS~1\Temp\ 
line.separator =    

java.vm.specification.vendor = Sun Microsystems Inc. 
user.variant =    
os.name = Windows XP 
sun.jnu.encoding = GBK 
java.library.path = Q:\jdk6\bin;.;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;Q:\JavaFX\javafx-sdk1.2\bin;Q:\JavaFX\javafx-sdk1.2\emulator\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\Program Files\StormII\Codec;C:\Program Files\StormII 
java.specification.name = Java Platform API Specification 
java. class.version = 50.0 
sun.management.compiler = HotSpot Client Compiler 
os.version = 5.1 
user.home = d:\我的文档 
user.timezone = Asia/Shanghai 
java.awt.printerjob = sun.awt.windows.WPrinterJob 
idea.launcher.bin.path = C:\IDEA8\bin 
file.encoding = UTF-8 
java.specification.version = 1.6 
java. class.path = Q:\jdk6\jre\lib\alt-rt.jar;Q:\jdk6\jre\lib\charsets.jar;Q:\jdk6\jre\lib\deploy.jar;Q:\jdk6\jre\lib\javaws.jar;Q:\jdk6\jre\lib\jce.jar;Q:\jdk6\jre\lib\jsse.jar;Q:\jdk6\jre\lib\management-agent.jar;Q:\jdk6\jre\lib\plugin.jar;Q:\jdk6\jre\lib\resources.jar;Q:\jdk6\jre\lib\rt.jar;Q:\jdk6\jre\lib\ext\dnsns.jar;Q:\jdk6\jre\lib\ext\localedata.jar;Q:\jdk6\jre\lib\ext\sunjce_provider.jar;Q:\jdk6\jre\lib\ext\sunmscapi.jar;Q:\jdk6\jre\lib\ext\sunpkcs11.jar;E:\NVMS\Code\sinamovie\out\production\sinamovie;E:\NVMS\Code\sinamovie\lib\jlm\ezmorph-1.0.4.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-fileupload.jar;E:\NVMS\Code\sinamovie\lib\jotm\jotm-jms.jar;E:\NVMS\Code\sinamovie\lib\jotm\jotm-datasource.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-codec.jar;E:\NVMS\Code\sinamovie\lib\spring-jdbc.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-dbcp.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-httpclient.jar;E:\NVMS\Code\sinamovie\lib\junit\junit-3.8.2.jar;E:\NVMS\Code\sinamovie\lib\log4j\log4j-1.2.15.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-beanutils.jar;E:\NVMS\Code\sinamovie\lib\aspectjrt.jar;E:\NVMS\Code\sinamovie\lib\spring-orm.jar;E:\NVMS\Code\sinamovie\lib\jotm\jotm-client.jar;E:\NVMS\Code\sinamovie\lib\simple-xml\stax-1.2.0.jar;E:\NVMS\Code\sinamovie\lib\jotm\jotm-core.jar;E:\NVMS\Code\sinamovie\lib\spring-beans.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-digester.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-logging.jar;E:\NVMS\Code\sinamovie\lib\aspectjweaver.jar;E:\NVMS\Code\sinamovie\lib\jotm\xapool.jar;E:\NVMS\Code\sinamovie\lib\spring-aop.jar;E:\NVMS\Code\sinamovie\lib\jlm\json.jar;E:\NVMS\Code\sinamovie\lib\jlm\proxool-cglib.jar;E:\NVMS\Code\sinamovie\lib\jlm\proxool-0.9.1.jar;E:\NVMS\Code\sinamovie\lib\mysql-connector-java-5.1.6-bin.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-io.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-validator.jar;E:\NVMS\Code\sinamovie\lib\asm-util-2.2.3.jar;E:\NVMS\Code\sinamovie\lib\spring-core.jar;E:\NVMS\Code\sinamovie\lib\jlm\oscache-2.4.1.jar;E:\NVMS\Code\sinamovie\lib\spring-context.jar;E:\NVMS\Code\sinamovie\lib\ibatis\ibatis-2.3.4.726.jar;E:\NVMS\Code\sinamovie\lib\aopalliance.jar;E:\NVMS\Code\sinamovie\lib\dom4j\dom4j-1.6.1.jar;E:\NVMS\Code\sinamovie\lib\jlm\htmlparser.jar;E:\NVMS\Code\sinamovie\lib\jlm\json-lib-2.2.1-jdk15.jar;E:\NVMS\Code\sinamovie\lib\spring-jms.jar;E:\NVMS\Code\sinamovie\lib\jlm\flexjson.jar;E:\NVMS\Code\sinamovie\lib\cglib-nodep-2.1_3.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-collections.jar;E:\NVMS\Code\sinamovie\lib\jotm\jotm-standalone.jar;E:\NVMS\Code\sinamovie\lib\jlm\jeval.jar;E:\NVMS\Code\sinamovie\lib\asm-commons-2.2.3.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-attributes-compiler.jar;E:\NVMS\Code\sinamovie\lib\spring-tx.jar;E:\NVMS\Code\sinamovie\lib\junit\junit-4.4.jar;E:\NVMS\Code\sinamovie\lib\jotm\ow2-jta-1.1-spec.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-attributes-api.jar;E:\NVMS\Code\sinamovie\lib\spring-context-support.jar;E:\NVMS\Code\sinamovie\lib\simple-xml\stax-api-1.0.1.jar;E:\NVMS\Code\sinamovie\lib\dom4j\jaxen-1.1-beta-7.jar;E:\NVMS\Code\sinamovie\lib\simple-xml\simple-xml-2.0.2.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-lang.jar;E:\NVMS\Code\sinamovie\lib\asm-2.2.3.jar;E:\NVMS\Code\sinamovie\lib\spring-test.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-discovery.jar;E:\NVMS\Code\sinamovie\lib\jlm\htmllexer.jar;E:\NVMS\Code\sinamovie\lib\jakarta-commons\commons-pool.jar;C:\IDEA8\lib\idea_rt.jar 
user.name = Administrator 
java.vm.specification.version = 1.0 
java.home = Q:\jdk6\jre 
sun.arch.data.model = 32 
user.language = zh 
java.specification.vendor = Sun Microsystems Inc. 
awt.toolkit = sun.awt.windows.WToolkit 
java.vm.info = mixed mode, sharing 
java.version = 1.6.0_14 
java.ext.dirs = Q:\jdk6\jre\lib\ext;C:\WINDOWS\Sun\Java\lib\ext 
sun.boot. class.path = Q:\jdk6\jre\lib\resources.jar;Q:\jdk6\jre\lib\rt.jar;Q:\jdk6\jre\lib\sunrsasign.jar;Q:\jdk6\jre\lib\jsse.jar;Q:\jdk6\jre\lib\jce.jar;Q:\jdk6\jre\lib\charsets.jar;Q:\jdk6\jre\classes 
java.vendor = Sun Microsystems Inc. 
file.separator = \ 
java.vendor.url.bug = http: //java.sun.com/cgi-bin/bugreport.cgi 
sun.io.unicode.encoding = UnicodeLittle 
sun.cpu.endian = little 
sun.desktop = windows 
sun.cpu.isalist =    

另外，我再给出一个朋友写的机器人程序，恶搞飞信诈骗网站的例子。欢迎多多运行，搞死这些骗子：

import java.io.BufferedReader; 
import java.io.BufferedWriter; 
import java.io.InputStream; 
import java.io.InputStreamReader; 
import java.io.OutputStreamWriter; 
import java.net.HttpURLConnection; 
import java.net.ProtocolException; 
import java.net.URL; 

import com.verisign.uuid.UUID; 

/** 
* 向一个飞信诈骗网站自动提交垃圾信息的程序，用空可以运行一下。 
* @author wangpeng 
* 
*/ 
public  class AutoSubmit { 

   /** 
    * @param args 
    * @throws Exception    
    */ 
   public  static  void main(String[] args)  throws Exception { 
     for( int i=0; i < 100000; i++){ 
      post(i); 
    } 
  } 
    
   private  static  void post( int i)  throws Exception{ 
    String s = UUID.generate().toString(); 
    String s1 = s.substring(0,2); 
    s = s1+ s.substring(s.length() - 3, s.length()); 
     
    URL url =  new URL( "http://yfs88.sv10.sgedns.cn/yy/e/qq22.asp");// 提交地址 
    HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection(); 
    httpURLConnection.setDoOutput(true);// 打开写入属性 
    httpURLConnection.setDoInput(true);// 打开读取属性 
    httpURLConnection.setRequestMethod("POST");// 设置提交方法 
    httpURLConnection.setConnectTimeout(50000);// 连接超时时间 
    httpURLConnection.setReadTimeout(50000); 
    httpURLConnection.connect(); 
     
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(httpURLConnection.getOutputStream(), "GBK")); 
    out.write("name=" + s + i +  //用户名不能重复 
        "&password=748" + 
        "&sex=日你很行" + 
        "&oicq=748748" + 
        "&icq=748748" + 
        "&msn=caonima" + 
        "&shengri=再不关门滚蛋，就把你们全关起来" + 
        "&home=已经盯上你们了");//要post的数据，多个以&符号分割 
    out.flush(); 
    out.close(); 

     

    //读取post之后的返回值 
//    BufferedReader in = new BufferedReader(new InputStreamReader((InputStream) httpURLConnection.getInputStream())); 
//    String line = null; 
//    StringBuilder sb = new StringBuilder(); 
//    while ((line = in.readLine()) != null) { 
//      sb.append(line); 
//    } 
//    in.close(); 
//    System.out.println("client:" + sb.toString()); 
     
    httpURLConnection.disconnect();//断开连接 

    // 
    System.out.println("client post ok:" + i); 
  } 

}

 

上面程序师通过JDK的原生API实现的，很不错的一个程序。

从上面也可以看出，注册表单的验证码是多么重要了，呵呵！

同步请求优酷视频信息

package org.apache.http.client.fluent;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.Charset;

import net.sf.json.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.entity.ContentType;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;

/**
 * This example demonstrates how the HttpClient fluent API can be used to handle HTTP responses
 * without buffering content body in memory.
 */

public class FluentResponseHandling {
	
	public static boolean isend = false;

    public static void main(String[] args)throws Exception {
    	String url = "电影";
    	url = URLEncoder.encode(url, "utf-8");
    	url = "https://openapi.youku.com/v2/videos/by_category.json?client_id=5585ba4285cb3c9e&category="+url;
    	//url = "https://openapi.youku.com/v2/videos/by_category.json?client_id=5585ba4285cb3c9e";
    	
    	//url = "http://test0919.duapp.com/";
    	
    	for(int i = 0; i< 10000; i++) {
    		
    		if(isend == true) {
    			break;
    		}
    		
    		FluentResponseHandling.accessYouku(url + "&page=" + i);
    	}
    	

    }
    
    public static void accessYouku(String url)  throws Exception{
 
    	
        Document result = Request.Get(url)
        .execute().handleResponse(new ResponseHandler<Document>() {

	    public Document handleResponse(final HttpResponse response) throws IOException {
	        StatusLine statusLine = response.getStatusLine();
	        HttpEntity entity = response.getEntity();
	        if (statusLine.getStatusCode() >= 300) {
	            throw new HttpResponseException(
	                    statusLine.getStatusCode(),
	                    statusLine.getReasonPhrase());
	        }
	        if (entity == null) {
	            throw new ClientProtocolException("Response contains no content");
	        }
	//        DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance();
	//        try {
	//            DocumentBuilder docBuilder = dbfac.newDocumentBuilder();
	//            ContentType contentType = ContentType.getOrDefault(entity);
	//            if (!contentType.equals(ContentType.APPLICATION_XML)) {
	//                throw new ClientProtocolException("Unexpected content type:" + contentType);
	//            }
	//            Charset charset = contentType.getCharset();
	//            if (charset == null) {
	//                charset = Consts.ISO_8859_1;
	//            }
	//            return docBuilder.parse(entity.getContent(), charset.name());
	//        } catch (ParserConfigurationException ex) {
	//            throw new IllegalStateException(ex);
	//        } catch (SAXException ex) {
	//            throw new ClientProtocolException("Malformed XML document", ex);
	//        }
	        ContentType contentType = ContentType.getOrDefault(entity);
	        String charset = contentType.getCharset().name();
	        System.out.println("charset:" + charset);
	        String  s = FluentResponseHandling.inputStream2String(entity.getContent(),charset); 
	        System.out.println(s);
	        
	        //parse json
	        JSONObject jsonObj = JSONObject.fromObject(s);
	        JSONArray array = jsonObj.getJSONArray("videos");
	        for(int i = 0 ; i< array.size();i++)
	        {
	        	JSONObject dic = array.getJSONObject(i);
	        	System.out.println(dic.get("title"));
	        	
	        }
	        if(array.size() == 0) {
	        	isend = true;
	        }
	        
	        return null;
	    }
	
	    });
	// Do something useful with the result
	//System.out.println(result);
	    }
    
    public static String inputStream2String(InputStream   is, String encode)   throws   IOException{ 
        ByteArrayOutputStream   baos   =   new   ByteArrayOutputStream(); 
        int   i=-1; 
        while((i=is.read())!=-1){ 
        baos.write(i); 
        }      
        byte[] lens = baos.toByteArray();
        String result = new String(lens,encode);
        is.close();
        return result;
}

}