Nutch抓取源码分析之Injector类

(1) 将URL集合进行格式化和过滤，消除其中的非法URL，并设定URL状态(UNFETCHED),按照一定方法进行初始化分值；

(2) 将URL进行合并，消除重复的URL入口；

(3) 将URL及其状态、分值存入crawldb数据库，与原数据库中重复的则删除旧的，更换新的。

 

inject方法

 

public void inject(Path crawlDb, Path urlDir) throws IOException {
    //创建临时目录
    Path tempDir = new Path(getConf().get("mapred.temp.dir", ".") +
               "/inject-temp-"+
               Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
   // map text input file to a <url,CrawlDatum> file
    JobConf sortJob = new NutchJob(getConf());
    sortJob.setJobName("inject " + urlDir);
    FileInputFormat.addInputPath(sortJob, urlDir);
    sortJob.setMapperClass(InjectMapper.class);
    FileOutputFormat.setOutputPath(sortJob, tempDir);
    sortJob.setOutputFormat(SequenceFileOutputFormat.class);
    sortJob.setOutputKeyClass(Text.class);
    sortJob.setOutputValueClass(CrawlDatum.class);
    sortJob.setLong("injector.current.time", System.currentTimeMillis());
    JobClient.runJob(sortJob);

    // merge with existing crawl db
    JobConf mergeJob = CrawlDb.createJob(getConf(), crawlDb);
    FileInputFormat.addInputPath(mergeJob, tempDir);
    mergeJob.setReducerClass(InjectReducer.class);
    JobClient.runJob(mergeJob);
    CrawlDb.install(mergeJob, crawlDb);
    // 删除临时文件
    FileSystem fs = FileSystem.get(getConf());
    fs.delete(tempDir, true);
    long end = System.currentTimeMillis();
}

InjectMapper类

public static class InjectMapper implements Mapper<WritableComparable, Text, Text, CrawlDatum> {
    private URLNormalizers urlNormalizers; //URL 标准化工具
    private int interval;   //设置抓取间隔时间
    private float scoreInjected; //url 对页面的得分值
    private JobConf jobConf;
    private URLFilters filters;    //url 过滤器
    private ScoringFilters scfilters;  //得分器
    private long curTime;

    public void configure(JobConf job) {
      this.jobConf = job;
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      interval = jobConf.getInt("db.fetch.interval.default", 2592000);
      filters = new URLFilters(jobConf);
      scfilters = new ScoringFilters(jobConf);
      scoreInjected = jobConf.getFloat("db.score.injected", 1.0f);
      curTime = job.getLong("injector.current.time", System.currentTimeMillis());
    }
    public void close() {}
    public void map(WritableComparable key, Text value,
                    OutputCollector<Text, CrawlDatum> output, Reporter reporter)
      throws IOException {
      String url = value.toString();              // value is line of text

      if (url != null && url.trim().startsWith("#")) {
                 return;
      }

      // if tabs : metadata that could be stored
      // must be name=value and separated by \t
      float customScore = -1f;
      int customInterval = interval;
      Map<String,String> metadata = new TreeMap<String,String>();
      if (url.indexOf("\t")!=-1){
          String[] splits = url.split("\t");
          url = splits[0];
          for (int s=1;s<splits.length;s++){
              // find separation between name and value
              int indexEquals = splits[s].indexOf("=");
              if (indexEquals==-1) {
                  // skip anything without a =
                  continue;
              }
              String metaname = splits[s].substring(0, indexEquals);
              String metavalue = splits[s].substring(indexEquals+1);
              if (metaname.equals(nutchScoreMDName)) {
                  try {
                  customScore = Float.parseFloat(metavalue);}
                  catch (NumberFormatException nfe){}
              }
              else if (metaname.equals(nutchFetchIntervalMDName)) {
                  try {
                      customInterval = Integer.parseInt(metavalue);}
                  catch (NumberFormatException nfe){}
              }
              else metadata.put(metaname,metavalue);
          }
      }
      try {
        //url 标准化
        url = urlNormalizers.normalize(url, URLNormalizers.SCOPE_INJECT);
        //过滤非法url
        url = filters.filter(url);
      } catch (Exception e) {
        if (LOG.isWarnEnabled()) { LOG.warn("Skipping " +url+":"+e); }
        url = null;
      }
      if (url != null) {                          // if it passes
        value.set(url);                           // collect it
       // CrawlDatum保存注入状态、抓取间隔时间，抓取时间、得分等等
        CrawlDatum datum = new CrawlDatum(CrawlDatum.STATUS_INJECTED, customInterval);
        datum.setFetchTime(curTime);
        // now add the metadata
        Iterator<String> keysIter = metadata.keySet().iterator();
        while (keysIter.hasNext()){
            String keymd = keysIter.next();
            String valuemd = metadata.get(keymd);
            datum.getMetaData().put(new Text(keymd), new Text(valuemd));
        }
        if (customScore != -1) datum.setScore(customScore);
        else datum.setScore(scoreInjected);
        try {
            scfilters.injectedScore(value, datum);
        } catch (ScoringFilterException e) {
            if (LOG.isWarnEnabled()) {
                LOG.warn("Cannot filter injected score for url " + url
                        + ", using default (" + e.getMessage() + ")");
            }
        }
        output.collect(value, datum);
      }
    }
  }

 

InjectReducer 类

 

public static class InjectReducer implements Reducer<Text, CrawlDatum, Text, CrawlDatum> {
   public void configure(JobConf job) {}
   public void close() {}
   private CrawlDatum old = new CrawlDatum();
   private CrawlDatum injected = new CrawlDatum();

   public void reduce(Text key, Iterator<CrawlDatum> values,
                      OutputCollector<Text, CrawlDatum> output, Reporter reporter)
     throws IOException {
     boolean oldSet = false;
     while (values.hasNext()) {
       CrawlDatum val = values.next();
   // 如果某个URL已经注入到CrawlDB
       if (val.getStatus() == CrawlDatum.STATUS_INJECTED) {
         injected.set(val);
         //设置状态为不需要抓取
         injected.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
       } else {
         old.set(val);
         oldSet = true;
       }
     }
     CrawlDatum res = null;
     if (oldSet) res = old; // don't overwrite existing value
     else res = injected;
     output.collect(key, res);
   }
 }

转自：http://blog.csdn.net/cdl2008sky/article/details/7219836