python爬虫案例12个_Python 爬虫实例（12）—— python selenium 爬虫

# coding:utf-8

from common.contest import *def spider():

url = "http://www.salamoyua.com/es/subasta.aspx?origen=subastas&subasta=79"

chromedriver = 'C:/Users/xuchunlin/AppData/Local/Google/Chrome/Application/chromedriver.exe'chome_options =webdriver.ChromeOptions()

#使用代理

# proxies = r.get('4')

# chome_options.add_argument(('--proxy-server=http://' +proxies))

os.environ["webdriver.chrome.driver"] =chromedriver

driver = webdriver.Chrome(chromedriver, chrome_options=chome_options)

for i in range(1,100):

print "正在爬取第" + str(i) + "页的数据"

if i ==1:

# 请求url

driver.get(session_url)

result =driver.page_source

else:

try:

# 将页面滚动条拖到底部

js = "var q=document.documentElement.scrollTop=10000"driver.execute_script(js)

driver.find_element_by_id('ctl00_phContenidos_lbSiguiente').click()

# 得到爬取页面的结果

result =driver.page_source

time.sleep(3)

except:

result = ""soup = BeautifulSoup(result, 'html.parser')

result_div = soup.find_all('figure', attrs={"class": "Lotes fade"})

# print len(result_div)

for i inresult_div:

result_replace = replace(i)

print result_replace

item_url = re.findall('

item_imgurl = re.findall('

if "Remate" not in result_replace:

sold_price = ""

else:

sold_price = re.findall('

Remate:(.*?)

', result_replace)[0]

sold_price = sold_price.replace(' ','')

try:

item_lotnum = re.findall('title="Lote vendido">(.*?)', result_replace)[0]

item_lotnum = item_lotnum.replace('Lote','').replace(' ','')

except:

item_lotnum = re.findall('(.*?)',result_replace)[0]

item_lotnum = item_lotnum.replace('Lote', '').replace(' ', '')

print item_url

print item_lotnum

print item_imgurl

print sold_price

spider()