使用weblech爬取网页－配置文件
以下是在weblech目录下config文件夹中的配置文件Spider.properties
# Spider configuration file
#
# All of these settings default to sensible values if not specified.
# Directory in which to save downloaded files, defaults to "."
saveRootDirectory = c:/weblech/sites
# Filename in which to save mailto links
mailtoLogFile = mailto.txt
# Tell the spider to reload HTML pages each time, but not images
# or other files
refreshHTMLs = true
refreshImages = false
refreshOthers = false
# Set the extensions the Spider should use to determine which
# pages are of MIME type text/html. The Spider also learns new
# types as it downloads them.
htmlExtensions = htm,html,shtm,shtml
# Similarly for MIME type image/*
imageExtensions = gif,jpg,jpeg,png,bmp
# URL at which we should start the spider
startLocation = http://www.slashdot.org/
# Whether to do depth first search, or the default breadth
# first search when finding URLs to download
depthFirst = false
# Maximum depth of pages to retrieve (the first page is depth
# 0, links from there depth 1, etc). Setting to 0 is "unlimited"
maxDepth = 2
# Basic URL filtering. URLs must contain this string in order
# to be downloaded by WebLech
urlMatch = slashdot.org
# Basic URL prioritisation. URLs which are "interesting" are
# downloaded first, URLs which are "boring" last.
interestingURLs=pollBooth.pl,faq
boringURLs=article.pl
# User Agent header
userAgent = Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)
# Username and password for basic HTTP authentication, if required.
# The same username and password will be used for all authentication
# challenges during a download session.
basicAuthUser = myUser
basicAuthPassword = 1234
# Number of download threads to start
spiderThreads = 1
# How often to checkpoint the Spider. A checkpoint file is named
# "spider.checkpoint" and can be used to start the spider in the
# middle of a run. Setting this value to 0 disables checkpoints.
# Here we checkpoint every 30 seconds
checkpointInterval = 30000