【jsoup】使用jsoup爬取智联职业信息

首先我们引入jsoup的依赖

    org.jsoup
    jsoup
    1.7.3

然后我们编写爬虫类WebSpider

package cn.qblank.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 用于爬取一些职位信息,输出到某个文件中
 * @author evan_qb
 *
 */
public class WebSpider {
	public static  String getHtml(String url,String encoding) {
		URL uri = null;
		URLConnection conn = null;
		BufferedReader br = null;
		StringBuffer sb = new StringBuffer();
		try {
			//建立网络连接
			uri = new URL(url);
			//打开连接
			conn = uri.openConnection();
			//使用输入流读取网页数据
			InputStream in = conn.getInputStream();
			br = new BufferedReader(new InputStreamReader(in, encoding));
			String line = null;
			while((line = br.readLine()) != null) {
				sb.append(new String(line + "\n"));
			}
		} catch (Exception e) {
			e.printStackTrace();
		}finally {
			try {
				if (br != null) {
					br.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return sb.toString();
	}
	
	/**
	 * 获取对应内容
	 * @return
	 */
	public static List> getContent(String url,String encoding){
		//获取网页内容
		String html = getHtml(url, encoding);
		//将网页内容转成document的格式
		Document document = Jsoup.parse(html);
		//获取ul标签
		Elements elements = document.getElementsByClass("searchResultItemDetailed");
		
		//使用List

 

这样，使用jsoup写的第一个爬虫就完成了