自己写的实时爬取 CSDN 2012 博客之星 88位候选人排名
如果觉得我的技术文章还有点让列为看官汲取之处，
请给我投上宝贵的一篇，以兹鼓励呵，多谢，多谢！！
本人ID：m13666368773
投票地址：http://vote.blog.csdn.net/item/blogstar/m13666368773
有幸入选 CSDN 2012 博客之星 88位候选人，但是排名不是很靠前，想看看自己距离前面几名 多少投票，遂写了这个 粗劣的程序，跑了一下
原理：由于评选页面估计是 异步读取的信息，所以只能进入88名候选人投票页面，获取有用信息：用户名，票数，排名，所以爬虫爬行时间有点慢，需要优化，不过基本上实现排名。
程序如下：
package com.aptech;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.junit.Test;
@SuppressWarnings("unchecked")
public class TestPachongUrl {
static Map messageMap = new HashMap();
private static List list = new ArrayList();
static String url = "http://vote.blog.csdn.net/item/blogstar/";
static String user[] = new String[] { "Testing_is_believing", "t0nsha", "iukey", "yjflinchong", "taomanman", "chinafe", "hliq5399", "dog250", "qinjuning", "cheny_com", "v_JULY_v", "zhmxy555",
"Purpleendurer", "iihero", "yming0221", "ccanan", "tigerjb", "cheungmine", "hawksoft", "sheismylife", "hfahe", "cyq1984", "littletigerat", "kmyhy", "caimouse", "manoel", "xyz_lmn",
"hunkcai", "yiyaaixuexi", "norains", "clever101", "leftfist", "xiaominghimi", "niyi0318", "yanghuiliu", "abandonship", "mapdigit", "bill_man", "Augusdi", "LoveLion", "sunboy_2050",
"kongxx", "21aspnet", "chszs", "thl789", "mylxiaoyi", "akof1314", "yincheng01", "keyboardOTA", "pan_tian", "downmoon", "wangkuifeng0118", "robinson_0612", "bluishglc", "coolbacon",
"tangcheng_ok", "tianxiaode", "cjjky", "MoreWindows", "mr_raptor", "dojotoolkit", "chelsea", "chgaowei", "teamlet", "IBM_hoojo", "iefreer", "lee576", "jaminwm", "xuhuojun", "linghe301",
"caolaosanahnu", "ricohzhanglong", "totogo2010", "axman", "ce123", "rabbit729", "nkmnkm", "superdont", "m13666368773", "aomandeshangxiao", "hitlion2008", "siren0203", "feixiaoxing",
"Poechant", "cloudhsu", "Innost", "yanghua_kobe", "tianlesoftware" };
@Test
public static String test(URL url) throws Exception {
/**
* 首先要和URL下的URLConnection对话。 URLConnection可以很容易的从URL得到。比如： // Using
* java.net.URL and //java.net.URLConnection
*/
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
/**
* 然后把连接设为输出模式。URLConnection通常作为输入来使用，比如下载一个Web页。
* 通过把URLConnection设为输出，你可以把数据向你个Web页传送。下面是如何做：
*/
connection.setDoOutput(true);
connection.setRequestMethod("POST");
connection.setRequestProperty("user-agent", "mozilla/4.7 [en] (win98; i)");
connection.connect();
/**
* 最后，为了得到OutputStream，简单起见，把它约束在Writer并且放入POST信息中，例如： ...
*/
OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream(), "UTF-8");
out.flush();
out.close();
/**
* 这样就可以发送一个看起来象这样的POST： POST /jobsearch/jobsearch.cgi HTTP 1.0 ACCEPT:
* text/plain Content-type: application/x-www-form-urlencoded
* Content-length: 99 username=bob password=someword
*/
// 一旦发送成功，用以下方法就可以得到服务器的回应：
String sCurrentLine;
String sTotalString;
sCurrentLine = "";
sTotalString = "";
InputStream l_urlStream;
l_urlStream = connection.getInputStream();
// 传说中的三层包装阿！
BufferedReader l_reader = new BufferedReader(new InputStreamReader(l_urlStream));
while ((sCurrentLine = l_reader.readLine()) != null) {
sTotalString += sCurrentLine + "\r\n";
}
int begin0 = sTotalString.indexOf("博客地址：<a href=\"http://blog.csdn.net/");
int end0 = sTotalString.indexOf("\" class=\"red\" target=\"_blank\">");
int begin1 = sTotalString.indexOf("票数：<span class=\"red\">");
int end1 = sTotalString.indexOf("</span> 票</li>");
int begin2 = sTotalString.indexOf("当前排名：<span class=\"red\">");
int end2 = sTotalString.indexOf("</span> 名</li>");
String message = sTotalString.substring(begin0 + 35, end0) + "-" + sTotalString.substring(begin1 + 21, end1) + "=" + sTotalString.substring(begin2 + 23, end2);
return message;
}
public static void main(String[] args) throws Exception {
for (int i = 0; i < user.length; i++) {
list.add(new URL(url + user[i]));
}
SimpleDateFormat dateformat = new SimpleDateFormat("yyyy年MM月dd日 HH时mm分ss秒 E ");
String nowTime = dateformat.format(new Date());
System.out.println("统计时间：" + nowTime);
System.out.println("候选人数量：" + user.length);
System.out.println(addChinaBlank("用户名") + addChinaBlank("票数") + "排名");
for (int i = 0; i < list.size(); i++) {
String subMessage = test((URL) list.get(i));
String key = subMessage.substring(subMessage.indexOf("=") + 1, subMessage.length());
messageMap.put(key, subMessage);
}
for (int i = 1; i <= 88; i++) {
String endMessage = messageMap.get("" + i).toString();
System.out.println(addBlank(endMessage.substring(0, endMessage.indexOf("-"))) + endMessage.substring(endMessage.indexOf("-") + 1, endMessage.indexOf("=")) + " "
+ endMessage.substring(endMessage.indexOf("=") + 1, endMessage.length()));
}
}
public static String addBlank(String user) {
String blank = " ";
int userLength = user.length();
for (int i = 0; i < 30 - userLength; i++) {
user += blank;
}
return user;
}
public static String addChinaBlank(String message) {
String blank = " ";
int userLength = message.length() * 2;
for (int i = 0; i < 70 - userLength; i++) {
message += blank;
}
return message;
}
}
运行一下：
统计时间：2012年12月08日 13时31分24秒 星期六 候选人数量：88 用户名 票数 排名 v_JULY_v 1133 1 yincheng01 827 2 MoreWindows 446 3 mr_raptor 371 4 yiyaaixuexi 350 5 LoveLion 298 6 ricohzhanglong 295 7 tianlesoftware 243 8 xiaominghimi 236 9 taomanman 225 10 yming0221 205 11 zhmxy555 180 12 Poechant 165 13 aomandeshangxiao 146 14 linghe301 133 15 hawksoft 125 16 nkmnkm 112 17 cjjky 112 18 niyi0318 90 19 cyq1984 87 20 clever101 84 21 cloudhsu 79 22 akof1314 70 23 Testing_is_believing 65 24 cheny_com 57 25 yanghuiliu 52 26 lee576 45 27 manoel 45 28 bill_man 43 29 hfahe 43 30 tangcheng_ok 41 31 teamlet 41 32 dojotoolkit 41 33 cheungmine 41 34 yjflinchong 41 35 norains 40 36 sheismylife 38 37 m13666368773 36 38 coolbacon 36 39 pan_tian 34 40 sunboy_2050 34 41 qinjuning 31 42 Augusdi 30 43 21aspnet 29 44 tigerjb 29 45 axman 29 46 mapdigit 29 47 downmoon 27 48 chgaowei 27 49 ce123 26 50 mylxiaoyi 26 51 dog250 25 52 t0nsha 25 53 feixiaoxing 25 54 thl789 25 55 kongxx 25 56 abandonship 24 57 iukey 23 58 caimouse 22 59 caolaosanahnu 22 60 xyz_lmn 21 61 robinson_0612 20 62 IBM_hoojo 20 63 Innost 20 64 wangkuifeng0118 19 65 iihero 19 66 hunkcai 19 67 rabbit729 19 68 chelsea 19 69 totogo2010 18 70 tianxiaode 18 71 Purpleendurer 18 72 yanghua_kobe 17 73 jaminwm 16 74 iefreer 16 75 siren0203 16 76 ccanan 15 77 littletigerat 15 78 kmyhy 15 79 chszs 15 80 superdont 14 81 keyboardOTA 14 82 leftfist 14 83 chinafe 13 84 hliq5399 12 85 bluishglc 10 86 hitlion2008 9 87 xuhuojun 7 88