使用selenium爬取京东商城所有手机价格

直接上代码

package com.sbh.test.web;

import org.junit.Test;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;

import java.util.List;

public class Demo {

	@Test
	public void getCommodityInfo(){
		System.setProperty("webdriver.firefox.bin","D:\\app\\MozillaFirefox\\firefox.exe");
		FirefoxDriver driver = new FirefoxDriver();
		driver.get("https://www.jd.com/");
		WebElement key = driver.findElement(By.id("key"));
		key.click();
		key.sendKeys("手机");
		WebElement element = driver.findElement(By.xpath("/html/body/div[1]/div[3]/div/div[2]/div/div[2]/button/i"));
		element.click();
		while (true){
			String js="var q=document.documentElement.scrollTop=6000";
			driver.executeScript(js);
			try {
				Thread.sleep(10000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			WebElement j_goodsList = driver.findElementById("J_goodsList");
			WebElement element1 = j_goodsList.findElement(By.cssSelector("ul.gl-warp.clearfix"));
			//一个商品
			List li = element1.findElements(By.cssSelector("li>div.gl-i-wrap"));
			for (WebElement webElement : li) {
				String price=getPrice(webElement.findElement(By.cssSelector("div.p-price>strong")));
				String name=getName(webElement.findElement(By.cssSelector("div.p-name.p-name-type-2>a>em")));
				System.out.println(name);
				System.out.println(price);
			}
			//J_main
			WebElement pageNext = driver.findElement(By.cssSelector("div#J_bottomPage>span.p-num>a.pn-next"));
			String onclick = pageNext.getAttribute("onclick");
			if(onclick == null || onclick.isEmpty() ){
				break;
			}
			pageNext.click();
		}
	}

	/**
	 * 获取商品名称
	 * @param element
	 * @return
	 */
	private String getName(WebElement element) {

		return element.getText();
	}

	/**
	 * 获取商品价格
	 * @param element
	 * @return
	 */
	private String getPrice(WebElement element) {
		WebElement element2 = element.findElement(By.cssSelector("em"));
		WebElement element3 = element.findElement(By.cssSelector("i"));
		String text = element2.getText();
		String text1 = element3.getText();
		String s = text + text1;
		return s;
	}
}

转载于:https://my.oschina.net/u/3484671/blog/2988393