用HtmlUnit和httpClient抓施华洛世奇网站图片和动画<二>
try
{
mainPage = webClient.getPage(url);
} catch (Exception e)
{
log.error(e.getMessage(), e);
}
if (mainPage != null)
{
HtmlElement paginationContainer = mainPage
.getElementById("paginationContainer");
if (paginationContainer != null)
{
url = url + "/all-1";
try
{
mainPage = webClient.getPage(url);
} catch (Exception e)
{
log.error(e.getMessage(), e);
}
}
HtmlElement categories = mainPage.getElementById("categories");
if (categories == null)
{
HtmlElement products = mainPage.getElementById("products");
if(products!=null)
{
List<HtmlElement> productDivList = products.getElementsByTagName("div");
if(productDivList!=null&&productDivList.size()>0)
{
for(HtmlElement proDiv:productDivList)
{
List<HtmlElement> subList = proDiv.getElementsByTagName("div");
if(subList!=null)
{
for(HtmlElement dt:subList)
{
String classt = dt.getAttribute("class");
if (classt != null && classt.equals("productImg"))
{
List<HtmlElement> subAList = proDiv.getElementsByTagName("a");
if(subAList!=null&&subAList.size()>0)
{
HtmlElement ae=subAList.get(0);
String href = ae.getAttribute("href");
thirdPageUrls.add(href);
}
break;
}
}
}
}
}
}
}
else
{
List<HtmlElement> divList = categories.getElementsByTagName("div");
if (divList != null && divList.size() > 0)
{
for (HtmlElement div : divList)
{
List<HtmlElement> tempDivs = div
.getElementsByTagName("div");
if (tempDivs != null && tempDivs.size() > 0)
{
for (HtmlElement div1 : tempDivs)
{
List<HtmlElement> aList = div1
.getElementsByTagName("a");
HtmlAnchor a = (HtmlAnchor) aList
.get(0);
String link = a.getHrefAttribute();
secondPageUrls.add(link);
}
}
}
}
}
}
}
}
log.error("第二层抓取结束..........");
log.error("目前抓取到的第二层URL个数为:"+secondPageUrls.size());
int count=0;
for (String url : secondPageUrls)
{
count++;
log.error("正在抓取第二层的第"+count+"个URL:"+url);
HtmlPage mainPage = null;
try
{
mainPage = webClient.getPage(url);
} catch (Exception e)
{
log.error(e.getMessage(), e);
}
log.error("抓取URL完成:"+url+",正在分析URL"+url+"+结果的URL");
if (mainPage != null)
{
HtmlElement paginationContainer = mainPage
.getElementById("paginationContainer");
if (paginationContainer != null)
{
url = url + "/all-1";
}
HtmlElement products = mainPage.getElementById("products");
if (products != null)
{
List<HtmlElement> list = products
.getElementsByTagName("div");
;
if (list == null || list.size() == 0)
{
continue;
}
for (HtmlElement h : list)
{
String cls = h.getAttribute("class");
if (cls == null || !cls.equals("productName"))
{
continue;
}
List<HtmlElement> links = h.getElementsByTagName("a");
if (links != null && links.size() > 0)
{
HtmlAnchor htmlAnchor = (HtmlAnchor) links.get(0);
String linkStr = htmlAnchor.getHrefAttribute();
thirdPageUrls.add(linkStr);
log.error(linkStr);
}
}
}
}
}
log.error("第二层抓取结束..........");
secondPageUrls.clear();
secondPageUrls = null;
count=0;
log.error("目前抓取到的第三层URL个数为:"+thirdPageUrls.size());
String urlPrix="http://www.swarovski-crystallized.com/jewelry/us/";
for (String url : thirdPageUrls)
{
count++;
log.error("正在抓取第三层的第"+count+"个URL:"+url);
HtmlPage mainPage = null;
try
{
mainPage = webClient.getPage(url);
} catch (Exception e)
{
log.error(e.getMessage(), e);
}
if (mainPage != null)
{
log.error("抓取URL完成:"+url+",正在分析URL"+url+"+结果");
int indexC=url.indexOf(urlPrix);
int indexD=url.indexOf("?");
String dirStr=url.substring(indexC+urlPrix.length(), indexD);
String regEx = "/";
// Pattern p = Pattern.compile(regEx);
// Matcher m = p.matcher(dirStr);
//哦哦哦，建立文件夹准备把抓到数据放在里面
dirStr=replece( regEx,"\\\\",dirStr);
dirStr = "D:\\swaroski\\"+dirStr;
File file = new File(dirStr);
if(file.isDirectory())
{
dirStr=dirStr+"\\"+count;
file = new File(dirStr);
}
file.mkdirs();
Product product=new Product();
product.setLocalDir(dirStr);
product.setPageUrl(url);
HtmlElement rightCol = mainPage.getElementById("rightCol");
String title = null;
String description = null;
String packingUnit = null;
if (rightCol != null)
{
HtmlElement headlineDiv = rightCol
.getElementById("headline");
if (headlineDiv != null)
。。。。。。。。。。。。。。。。。。。。。。。。
log.error("完成:"+url+",分析结果");
try
{
swaroSkiDAO.addProduct(product);
} catch (Exception e)
{
log.error(e.getMessage(), e);
}
log.error("完成保存结果");
for(String downloadUrl:resourceUrlList)
{
int index6=downloadUrl.lastIndexOf("/");
String fileName=downloadUrl.substring(index6+1);
String dirStr2=dirStr+"\\"+fileName;
File storeFile = new File(fileName);
if(storeFile.exists())
{
continue;
}
SaveFileThread runable=new SaveFileThread(dirStr2,downloadUrl,sem);
pools.submit(runable);
log.error("开始提交下载文件:"+downloadUrl);
try
{
Thread.sleep(2500);
} catch (InterruptedException e)
{
}
}
//啊哈哈，这个就是去拿下图片和动画的线程，结束
class SaveFileThread implements Runnable
{
private String fileName;
private String downloadUrl;
private Semaphore sem;
public SaveFileThread(String fileName,String downloadUrl,Semaphore sem)
{
this.fileName=fileName;
this.downloadUrl=downloadUrl;
this.sem=sem;
}
public void run()
{
HttpClient client = new HttpClient();
GetMethod get = new GetMethod(downloadUrl);
FileOutputStream output=null;
try
{
client.executeMethod(get);
File storeFile = new File(fileName);
output = new FileOutputStream(storeFile);
output.write(get.getResponseBody());
output.flush();
} catch (Exception e)
{
log.error(e.getMessage(), e);
}