2.用scrapy 爬取链家网站 全国的二手房信息。

# endcoding:utf-8
import scrapy,time,random,re
要使用该爬虫在命令行输入以下命令即可。
# scrapy runspider quotes_spider.py -o quotes.json 
class QuotesSpider(scrapy.Spider):
	# allowed_domains='lianjia.com'
	name = 'lianjia_ershou'
	start_urls=['https://www.lianjia.com/city/']
	headers={
	'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
	}
	def start_requests(self):
		for url in self.start_urls:
			yield scrapy.Request(url,callback=self.parse_city,headers=self.headers)

	def parse_city(self,response):
		urls=response.css('.city_list a::attr("href")').getall()
		urls=[url+'ershoufang/' for url in urls]
		for url in urls:
			yield scrapy.Request(url, self.parse_quyu,headers=self.headers)
	def parse_quyu(self,response):
		# print('=='*50)
		urls=response.xpath('//div[@data-role="ershoufang"]/div[1]/a/@href').getall()
		for url in urls:
			url=response.urljoin(url)
			if url!=response.url:
				print(url)
				yield scrapy.Request(url,self.parse,headers=self.headers)
		
	def parse(self, response):
		
		data=response.xpath('//div[@class="info clear"]')
		titles=data.css('.title a::text').getall()
		address=data.css('.address .houseInfo a::text').getall()
		houseInfo=data.css('.address .houseInfo::text').getall()
		floods=data.css('.flood').xpath('div/text()').getall()

		priceInfos=data.css('.priceInfo')
		totalPrices=[i+'万' for i in data.css('.totalPrice span::text').getall()]
		unitPrices=priceInfos.css('.unitPrice span::text').getall()
		for title,address,houseInfo,flood,totalPrice,unitPrice in zip(titles,address,houseInfo,floods,totalPrices,unitPrices):
			yield{
				'title':title,
				'address':address,
				'houseInfo':houseInfo,
				'flood':flood,
				'totalPrice':totalPrice,
				'unitPrice':unitPrice,
			}

		resp=response.xpath('//div[@class="page-box house-lst-page-box"]')
		try:
			totalPage=resp.re(r'totalPage":(\d+)')[0]
			curPage=resp.re(r'curPage":(\d+)}')[0]
		except Exception as e:
			print(e)
			return 

		# https://sz.lianjia.com/ershoufang/luohuqu/pg2/
		time.sleep((random.random()+0.5)*2)
		if  int(curPage)!=int(totalPage):
			if not 'pg' in response.url:
				url1=response.url+'pg'+str(int(curPage)+1)+'/'
			else:
				# url1=response.url+'pg'+str(int(curPage)+1)+'/'
				url1=re.sub(re.compile(r"\d+", re.S), str(int(curPage)+1), response.url)

			# print(url1)
			# exit()
			yield scrapy.Request(url1,callback=self.parse,headers=self.headers)