java爬图片数据 demo

package com.xcx.spots.test;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import us.codecraft.webmagic.selector.Selectable;
public class Danli implements PageProcessor {

private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

@Override
public void process(Page page) {
    //如果页面是列表页
    if (page.getUrl().toString().equals("http://www.mmonly.cc/ktmh/dmmn/")) {

        //得到列表图片的所有链接
        page.addTargetRequests(page.getHtml().$("div.item_t > div > div.ABox > a").links().all());

        // 获取下一页，倒数第个a标签
        // 规定属于其父元素的第二个子元素的每个 p 元素，从最后一个子元素开始计数：
        //这样就拿到了下一页的link
        page.addTargetRequest(page.getHtml().$("#pageNum > a:nth-last-child(2)").links().toString());

        //详情页                                                   匹配括号内的数字
    } else if (page.getUrl().regex("http://www.mmonly.cc/ktmh/dmmn/[\\d]+") != null) {
        // 得到详情页里面的下一页按钮的  href
        Selectable links = page.getHtml().$("#nl > a").links();

        if (links != null )
            page.addTargetRequest(links.toString());
        // 抓取图片内容p标签
        String img = page.getHtml().$("#big-pic p img").toString();
        //细查a标签
        if (img == "null")
            //img标签直接使用link()不能直接获取，所以只能获取到内容，进行内容的截取
            img = page.getHtml().$("#big-pic a img").toString();
        //截取从s开始计，+5就刚好到h的位置
        img = img.substring(img.indexOf("src=\"") + 5, img.length() - 2);

        page.putField("img", img);



    }
}

@Override
public Site getSite() {
    return site;
}

public static void main(String[] args) {
    Spider.create(new Danli())
            .addUrl("http://www.mmonly.cc/ktmh/dmmn/")
            .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000)))
            .addPipeline(new Demo()).thread(5).run();
}

}
package com.xcx.spots.test;

/**

• @ProjectName: spots

• @Package: com.xcx.spots.test

• @ClassName: Demo

• @Author: nh

• @Description:

• @Date: 2020/7/2 13:48

• @Version: 1.0
  */
  import us.codecraft.webmagic.ResultItems;
  import us.codecraft.webmagic.Task;
  import us.codecraft.webmagic.pipeline.Pipeline;
  public class Demo implements Pipeline {
  @Override
  public void process(ResultItems resultItems, Task task) {

   String url = resultItems.get("img").toString();
   UrlFileDownloadUtil.downloadPicture(url);
  

  }
  }
  package com.xcx.spots.test;

/**

• @ProjectName: spots
• @Package: com.xcx.spots.test
• @ClassName: UrlFileDownloadUtil
• @Author: nh
• @Description:
• @Date: 2020/7/2 13:48
• @Version: 1.0
  /
  import java.io.;
  import java.net.URL;
  import java.util.UUID;

public class UrlFileDownloadUtil {

public static void downloadPicture(String url) {
    String file = "D:\\image\\";
    try {
        URL u = new URL(url);
        String name = UUID.randomUUID().toString();

        DataInputStream inputStream = new DataInputStream(u.openStream());
        FileOutputStream outputStream = new FileOutputStream(file+name+".jpg");

        byte [] bytes = new byte[1024*100];
        int length ;
        while ((length=inputStream.read(bytes))>0){
            outputStream.write(bytes,0,length);
        }
        System.out.println("下载完成："+file+name+".jpg");

        inputStream.close();
        outputStream.close();
    } catch (  Exception e) {
        e.printStackTrace();
    }
}

}