nutch fetcher.server.delay

1 配置因素

<property>
  <name>fetcher.server.delay</name>
  <value>0.0</value>
  <description>The number of seconds the fetcher will delay between
   successive requests to the same server.</description>
</property>

 

2 机器人协议因素

FetchItemQueue fiq = fetchQueues.getFetchItemQueue(fit.queueID);

                  fiq.crawlDelay = rules.getCrawlDelay();

                  if (LOG.isDebugEnabled()) {

                    LOG.info("Crawl delay for queue: " + fit.queueID + " is set to " + fiq.crawlDelay + " as per robots.txt. url: " + fit.url);

                  }