在网上查了好长时间,发现httpclient的例子都是讲httpclient 3.X的。httpclient 4.0的例子简直是凤毛麟角。只好自己上官网仔细了查了下文档和官网示例,受益匪浅啊!
由于前段时间对“开心网”的“开心大亨”很感兴趣,但是每隔一段时间就需要看一下最新的价格变化,很是麻烦,由于我就想起用httpclient写个程序,自动查看最新的价格,并和前次的价格做比较,好让我可以选择利润最大的进行投资。
程序用到的包如下,为了附件的瘦身,请大家自己下载啊,附件中是没有这些lib的啊。
commons-collections-3.2.1.jar
commons-configuration-1.6.jar
commons-io-1.4.jar
commons-lang-2.4.jar
commons-logging-1.1.1.jar
httpclient-4.0.jar
httpcore-4.0.1.jar
由于只是例子,而且httpclient4.0的结构变化很大,自己对其理解也不是很深,此程序只是个入门程序,如果有高手在的话,希望大家可以提出宝贵的意见。
程序写的不是很美观,但意在让大家明白如何简单的使用httpclient4.0,所以没有对程序做任何的重构的整理,望大家多多谅解啊。
程序代码如下:
主程序: KaiXin001.java
/* * 2009/12/21 * Author: Yuan Hongzhi */ import java.io.File; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.cookie.Cookie; import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; import org.apache.http.util.EntityUtils; public class KaiXin001 { public InputStream getResourceAsStream(String filename) throws Exception { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); InputStream in = null; if (classLoader != null) { in = classLoader.getResourceAsStream(filename); } if (in == null) { in = ClassLoader.getSystemResourceAsStream(filename); } if (in == null) { throw new Exception("Can't find resource file: " + filename); } else { return in; } } public void writeToFile(String file, HttpEntity entity) throws Exception { writeToFile(file, EntityUtils.toString(entity)); } public void writeToFile(String file, String data) throws Exception { FileUtils.writeStringToFile(new File(file), data, "UTF-8"); } public String getContent(HttpEntity entity) throws Exception { if (entity != null) { entity = new BufferedHttpEntity(entity); long len = entity.getContentLength(); System.out.println("Length: " + len); System.out.println("====================\r\n"); return EntityUtils.toString(entity, "UTF-8"); } else { System.out.println("entity is null."); return null; } } public void setCookie(DefaultHttpClient httpclient, List<Cookie> cookies) { if (cookies.isEmpty()) { System.out.println("Cookie is empty."); return; } else { for (int i = 0; i < cookies.size(); i++) { System.out.println((i + 1) + " - " + cookies.get(i).toString()); httpclient.getCookieStore().addCookie(cookies.get(i)); } System.out.println(); } } // "开心网其它组件URL如下,大家可以添加上自己喜欢的组件URL。" // "http://www.kaixin001.com/!slave/index.php", "朋友买卖" // "http://www.kaixin001.com/!parking/index.php", "争车位" // "http://www.kaixin001.com/!house/index.php?_lgmode=pri", "买房子" // "http://www.kaixin001.com/!house/index.php?_lgmode=pri&t=49" // "http://www.kaixin001.com/!house/garden/index.php","花园" // "http://www.kaixin001.com/!rich/market.php", "超级大亨" public String enterComponentContent(String url, String componentName, DefaultHttpClient httpclient, List<Cookie> cookies, HttpResponse response, HttpEntity entity) throws Exception { System.out.println("--- Enter: " + componentName + " ---"); System.out.println("--- Url: " + url + " ---"); setCookie(httpclient, cookies); HttpGet httpget = new HttpGet(url); response = httpclient.execute(httpget); entity = response.getEntity(); return getContent(entity); } public void showResponseStatus(HttpResponse response) { // System.out.println(response.getProtocolVersion()); // System.out.println(response.getStatusLine().getStatusCode()); // System.out.println(response.getStatusLine().getReasonPhrase()); System.out.println(response.getStatusLine().toString()); System.out.println("-------------------------\r\n"); } public static void main(String[] args) throws Exception { KaiXin001 kx = new KaiXin001(); DefaultHttpClient httpclient = new DefaultHttpClient(); HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php"); List<NameValuePair> qparams = new ArrayList<NameValuePair>(); qparams.add(new BasicNameValuePair("email", "email")); qparams.add(new BasicNameValuePair("password", "password")); httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8)); HttpResponse response = httpclient.execute(httpost); kx.showResponseStatus(response); // HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); kx.getContent(entity); // Login List<Cookie> cookies = httpclient.getCookieStore().getCookies(); System.out.println("Post logon cookies:"); kx.setCookie(httpclient, cookies); // Redirect to home page String homepage = "http://www.kaixin001.com/home/"; String content = null; content = kx.enterComponentContent(homepage, "Home page", httpclient, cookies, response, entity); // Component String componet = "http://www.kaixin001.com/!rich/market.php"; content = kx.enterComponentContent(componet, "Component", httpclient, cookies, response, entity); // -------------------------------------------- kx.writeToFile("c:/kaixin.html", content); // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
访问开心大亨组件的程序:KaiXin_Rich.java
/* * 2009/12/21 * Author: Yuan Hongzhi */ import java.awt.Toolkit; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.collections.keyvalue.DefaultKeyValue; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpPost; import org.apache.http.cookie.Cookie; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HTTP; public class KaiXin_Rich extends KaiXin001 { public static List<DefaultKeyValue> curList = new ArrayList<DefaultKeyValue>(); public static List<DefaultKeyValue> preList = new ArrayList<DefaultKeyValue>(); public void copyList(List<DefaultKeyValue> src, List<DefaultKeyValue> des) { for (DefaultKeyValue dkv : src) { DefaultKeyValue kv = new DefaultKeyValue(dkv.getKey(), dkv.getValue()); des.add(kv); } } public Double subNumber(Object string) { String rex = "([\\d\\.]*)"; Pattern pattern = Pattern.compile(rex); Matcher match = pattern.matcher(string.toString()); if (match.find()) { return Double.valueOf(match.group()); } else return -1.0; } public String extractNoneAscii(Object string) { String rex = "([^\\x00-\\xff]+)"; Pattern pattern = Pattern.compile(rex); Matcher match = pattern.matcher(string.toString()); if (match.find()) { return match.group(); } else return ""; } public static void main(String[] args) throws Exception { KaiXin_Rich kx = new KaiXin_Rich(); PropertiesConfiguration config = new PropertiesConfiguration(); config.load(kx.getResourceAsStream("properties/kaixin001.properties"), "UTF-8"); DefaultHttpClient httpclient = new DefaultHttpClient(); HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php"); List<NameValuePair> qparams = new ArrayList<NameValuePair>(); qparams.add(new BasicNameValuePair("email", config.getString("email"))); qparams .add(new BasicNameValuePair("password", config.getString("password"))); httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8)); HttpResponse response = httpclient.execute(httpost); kx.showResponseStatus(response); // HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); kx.getContent(entity); // Login List<Cookie> cookies = httpclient.getCookieStore().getCookies(); System.out.println("Post logon cookies:"); kx.setCookie(httpclient, cookies); // Redirect to home page String homepage = "http://www.kaixin001.com/home/"; String content = null; content = kx.enterComponentContent(homepage, "Home page", httpclient, cookies, response, entity); int loopTime = config.getInt("times"); long sleepTime = config.getLong("period") * 60; for (int j = 0; j < loopTime; j++) { // Component String componet = config.getString("url"); content = kx.enterComponentContent(componet, "Component", httpclient, cookies, response, entity); int beginIndex = content.indexOf(config.getString("startRex")); int endIndex = content.lastIndexOf(config.getString("endRex")); content = content.substring(beginIndex, endIndex); // -------------------------------------------- String rex = config.getString("displayKeyRex"); Pattern pattern = Pattern.compile(rex, Pattern.MULTILINE); Matcher match = pattern.matcher(content); List<String> keys = new ArrayList<String>(); String matched = null; while (match.find()) { matched = match.group(); keys.add(matched); } rex = config.getString("displayValueRex"); pattern = Pattern.compile(rex, Pattern.MULTILINE); match = pattern.matcher(content); List<String> values = new ArrayList<String>(); while (match.find()) { matched = match.group(); values.add(matched); } DefaultKeyValue kv = null; for (int i = 0; i < keys.size(); i++) { kv = new DefaultKeyValue(keys.get(i), values.get(i)); curList.add(kv); } String rtnString = ""; String eachLine = null; for (DefaultKeyValue dkv : curList) { eachLine = dkv.getKey() + "\t:" + dkv.getValue(); rtnString += eachLine; rtnString += "\r\n"; System.out.println(eachLine); } System.out.println("\r\n===== Changed =====\r\n"); String allPriceString = "===== Changed =====\r\n\r\n"; StringBuffer price = null; double changed = 0; boolean beepFlag = false; for (int i = 0; i < curList.size() && curList.size() == preList.size(); i++) { double curMoney = kx.subNumber(curList.get(i).getValue()); double preMoney = kx.subNumber(preList.get(i).getValue()); if ((curMoney - preMoney) > 1E-5) { changed = curMoney - preMoney; DecimalFormat format = new DecimalFormat("###.##"); String number = format.format(changed); price = new StringBuffer(); price.append(curList.get(i).getKey()); price.append("\t: "); price.append(preList.get(i).getValue()); price.append(" -> "); price.append(curList.get(i).getValue()); price.append("\t"); price.append(" Changed: "); price.append(number); price.append(kx.extractNoneAscii(curList.get(i).getValue())); System.out.println(price); allPriceString = allPriceString + price.toString() + "\r\n"; beepFlag = true; } } if (beepFlag) { Toolkit.getDefaultToolkit().beep(); } preList.clear(); // System.out.println(content); String dateTime = DateUtils .getSystemDateTime(DateTimePattern.DATE_TIME_LONG_UNFORMAT); kx.writeToFile("c:/kaixin001/price_" + dateTime + ".txt", rtnString + "\r\n" + allPriceString); // kx.writeToFile("c:/kaixin.html", content); kx.copyList(curList, preList); curList.clear(); Thread.sleep(sleepTime * 1000); } // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
个人签名
-------------------------------------