java httpclient 抓取网页 POST GET

httpclient post方法 以TTD数据库为例

//
//Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*   http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied.  See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation.  For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package cn.com.talkweb.http.client;

import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

public class log_http{

    public static void main(String[] args) throws Exception {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        try {
             //下面注释的是用get获取
// HttpGet httpGet = new HttpGet("http://bidd.nus.edu.sg/group/ttd/ttd.asp");
// CloseableHttpResponse response1 = httpclient.execute(httpGet);
// // The underlying HTTP connection is still held by the response object
// // to allow the response content to be streamed directly from the network socket.
// // In order to ensure correct deallocation of system resources
// // the user MUST either fully consume the response content or abort request
// // execution by calling CloseableHttpResponse#close().
//
// try {
// System.out.println(response1.getStatusLine());
// HttpEntity entity1 = response1.getEntity();
// // do something useful with the response body
// // and ensure it is fully consumed
// EntityUtils.consume(entity1);
// } finally {
// response1.close();
// }

            HttpPost httpPost = new HttpPost("http://bidd.nus.edu.sg/group/TTD/ZFTTDWholeSearch_disease.asp?SetQuery=Y");//这里很重要，这是提交的网址，不是当前页面的网址
            List <NameValuePair> nvps = new ArrayList <NameValuePair>();
            nvps.add(new BasicNameValuePair("KS", "cancer"));//输入内容，KS是对应的表单id
            //nvps.add(new BasicNameValuePair("password", "lipidong1990"));
            httpPost.setEntity(new UrlEncodedFormEntity(nvps));
            CloseableHttpResponse response2 = httpclient.execute(httpPost);
            //String login_success=response2.getFirstHeader("Location").getValue();

// HttpGet httpGet = new HttpGet("http://bidd.nus.edu.sg/group/TTD/ZFTTDWholeSearch_disease.asp");
// CloseableHttpResponse response1 = httpclient.execute(httpGet);
          // The underlying HTTP connection is still held by the response object
          // to allow the response content to be streamed directly from the network socket.
          // In order to ensure correct deallocation of system resources
          // the user MUST either fully consume the response content or abort request
          // execution by calling CloseableHttpResponse#close().

// try {
// System.out.println(response1.getStatusLine());
// HttpEntity entity1 = response1.getEntity();
// // do something useful with the response body
// // and ensure it is fully consumed
// System.out.println(EntityUtils.toString(entity1,"UTF-8"));
// EntityUtils.consume(entity1);
// } finally {
// response1.close();
// }

            try {
                System.out.println(response2.getStatusLine());

                HttpEntity entity2 = response2.getEntity();
               System.out.println("Response content: " + EntityUtils.toString(entity2,"UTF-8"));//打印html代码
                // do something useful with the response body
                // and ensure it is fully consumed
                EntityUtils.consume(entity2);
            } finally {
                response2.close();
            }
        } finally {
            httpclient.close();
        }
    }

}