利用Jsoup爬取百度搜索内容及链接生成对应Excel

最近有朋友遇到需求，说要整理百度搜索结果中的数据，在此小做了下demo仅供参考。

爬取结果：

话不多说，直接贴代码。

所需依赖：

    org.apache.poi
        poi-ooxml
        3.17
    
    
        org.apache.poi
        poi
        3.17
    
    
        org.jsoup
        jsoup
        1.11.3
    

具体代码如下：

package com.bdspilder.controller;

import com.bdspilder.entity.Message;
import com.bdspilder.utils.PoiNetUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

@RestController
public class SpiderController {

    @RequestMapping("/export")
    public void export(HttpServletResponse response){
        response.setContentType("octets/stream");
        String excelName = "百度搜索文件";
        try {
            List messageList = this.getBaiDuMessage(80, "防沉迷 site:4399.com");
            response.addHeader("Content-Disposition", "attachment;filename="+
                    new String(excelName.getBytes("gb2312"), "ISO8859-1" )+".xls");
            HSSFWorkbook wb = PoiNetUtils.export(excelName,messageList);
            ServletOutputStream out = response.getOutputStream();
            wb.write(out);
            out.flush();
            out.close();
            wb.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private List getBaiDuMessage(Integer pageNo,String search) throws IOException {
        List list = new ArrayList<>();
        //Integer pageNo = 10;
        for (int i = 0;i ","" )
                        .replaceAll("", "")
                        .replaceAll("&", " ");
                String url = a.get(0).attr("href");
                //加入list集合
                Message message = new Message();
                message.setMsg(msg);
                message.setUrl(url);
                list.add(message);
                //System.out.println(msg + "----- 链接：" + url);
            }
        }
        return list;
    }
}

package com.bdspilder.utils;
import com.bdspilder.entity.Message;
import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.ss.usermodel.HorizontalAlignment;
import java.util.List;
public class PoiNetUtils {
    /**
     *    生成对应excel
     * @param excelName 文件名
     * @param dataList  数据集
     * @return  工作簿
     */
    public static HSSFWorkbook export(String excelName, List dataList) {
        HSSFWorkbook wb = null;
        try {
            // 第一步，创建一个webbook，对应一个Excel文件
            wb = new HSSFWorkbook();
            //生成一个表格
            HSSFSheet sheet = wb.createSheet(excelName);
            // 第三步，在sheet中添加表头第0行
            HSSFRow row0 = sheet.createRow(0);
            // 第四步，创建单元格，并设置值表头 设置表头居中
            HSSFCellStyle style = wb.createCellStyle();
            style.setAlignment(HorizontalAlignment.CENTER); // 创建一个居中格式
            HSSFCell cell1 = row0.createCell(0);
            cell1.setCellValue("标题");
            cell1.setCellStyle(style);
            HSSFCell cell2 = row0.createCell(1);
            cell2.setCellValue("链接");
            cell2.setCellStyle(style);
            //第二行开始加载数据
            for (int i = 1;i

到此，只需访问对应的 http://xxxx/export 即可下载对应excel