网络爬虫

package com.heaton.bot;
import com.heaton.bot.*;
import java.net.*;
 
/**
 * The SpiderWorker class performs the actual work of
 * spidering pages. It is implemented as a thread
 * that is created by the spider class.
 *
 * Copyright 2001-2003 by Jeff Heaton (http://www.jeffheaton.com)
 *
 * @author Jeff Heaton
 * @version 1.2
 */
public class SpiderWorker extends Thread {
 
 /**
   * The URL that this spider worker
   * should be downloading.
   */
 protected String target;
 
 /**
   * The owner of this spider worker class,
   * should always be a Spider object.
   * This is the class that this spider
   * worker will send its data to.
   */
 protected Spider owner;
 
 /**
   * Indicates if the spider is busy or not.
   * true = busy
   * false = idle
   */
 protected boolean busy;
 
 /**
   * A descendant of the HTTP object that
   * this class should be using for HTTP
   * communication. This is usually the
   * HTTPSocket class.
   */
 protected HTTP http;
 
 /**
   * Constructs a spider worker object.
   *
   * @param owner The owner of this object, usually
   * a Spider object.
   * @param http
   */
 public SpiderWorker(Spider owner,HTTP http)
 {
    this.http = http;
    this.owner = owner;
 }
 
 /**
   * Returns true of false to indicate if
   * the spider is busy or idle.
   *
   * @return true = busy
   * false = idle
   */
 public boolean isBusy()
 {
    return this.busy;
 }
 
 /**
   * The run method causes this thread to go idle
   * and wait for a workload. Once a workload is
   * received, the processWorkload method is called
   * to handle the workload.
   */
 public void run()
 {
    for ( ;; ) {
      target = this.owner.getWorkload();
      if ( target==null )
        return;
      owner.getSpiderDone().workerBegin();
      processWorkload();
      owner.getSpiderDone().workerEnd();
    }
 }
 

 /**
   * The run method actually performs the
   * the workload assigned to this object.
   */
 public void processWorkload()
 {
    try {
      busy = true;
      Log.log(Log.LOG_LEVEL_NORMAL,"Spidering " + target );
      http.send(target,null);
      Attribute typeAttribute = http.getServerHeaders().get("Content-Type");
 
      // if no content-type at all, its PROBABLY not HTML
      if ( typeAttribute==null )
        return;
 
      // now check to see if is HTML, ONLY PARSE text type files(namely HTML)
      owner.processPage(http);     
      if ( !typeAttribute.getValue().startsWith("text/") )
        return;
 
      HTMLParser parse = new HTMLParser();
 
      parse.source = new StringBuffer(http.getBody());
      // find all the links
      while ( !parse.eof() ) {
        char ch = parse.get();
        if ( ch==0 ) {
          HTMLTag tag = parse.getTag();
          Attribute link = tag.get("HREF");
          if ( link==null )
            link = tag.get("SRC");
 
          if ( link==null )
            continue;
 
          URL target=null;
          try {
            target = new URL(new URL(this.target),link.getValue());
          } catch ( MalformedURLException e ) {
            Log.log(Log.LOG_LEVEL_TRACE,
                    "Spider found other link: " + link );
            owner.foundOtherLink(link.getValue());
            continue;
          }
 
          if ( owner.getRemoveQuery() )
            target = URLUtility.stripQuery(target);
          target = URLUtility.stripAnhcor(target);
 
 
          if ( target.getHost().equalsIgnoreCase(
                                                new URL(this.target).getHost()) ) {
            Log.log(Log.LOG_LEVEL_NORMAL,
                    "Spider found internal link: " + target.toString() );
            owner.foundInternalLink(target.toString());
          } else {
            Log.log(Log.LOG_LEVEL_NORMAL,
                    "Spider found external link: " + target.toString() );
            owner.foundExternalLink(target.toString());
          }
        }
      }
      owner.completePage(http,false);
 } catch ( java.io.IOException e ) {
      Log.log(Log.LOG_LEVEL_ERROR,
              "Error loading file("+ target +"): " + e );
      owner.completePage(http,true);
    } catch ( Exception e ) {
      Log.logException(
                      "Exception while processing file("+ target +"): ", e );
      owner.completePage(http,true);
    } finally {
      busy = false;
    }
 }
 
 /**
   * Returns the HTTP descendant that this
   * object should use for all HTTP communication.
   *
   * @return An HTTP descendant object.
   */
 public HTTP getHTTP()
 {
    return http;
 }
}