scrapy setting.py 配置

 # Scrapy settings for proxy project
 2 #
 3 # For simplicity, this file contains only the most important settings by
 4 # default. All the other settings are documented here:
 5 #
 6 #     http://doc.scrapy.org/topics/settings.html
 7 #
 8 
 9 BOT_NAME = 'proxy'
10 BOT_VERSION = '1.0'
11 
12 SPIDER_MODULES = ['proxy.spiders']
13 NEWSPIDER_MODULE = 'proxy.spiders'
14 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
15 
16 DOWNLOAD_DELAY = 0
17 DOWNLOAD_TIMEOUT = 30
18 
19 ITEM_PIPELINES = [
20     'proxy.pipelines.ProxyPipeline'
21 ]
22 CONCURRENT_ITEMS = 100
23 CONCURRENT_REQUESTS_PER_SPIDER = 64
24 CONCURRENT_SPIDERS = 128
25 
26 LOG_ENABLED = True
27 LOG_ENCODING = 'utf-8'
28 LOG_FILE = '/home/xxx/services_runenv/crawlers/proxy/proxy/log/proxy.log'
29 LOG_LEVEL = 'DEBUG'
30 LOG_STDOUT = False