HttpClient爬虫-搜狗微信出现验证码

---

前言

在搜狗微信输入关键字搜索，请求频繁时会出现验证码页面

更换新的cookie即可解决

---

代码

		// String url = "https://weixin.sogou.com/weixin?type=2&s_from=input&query=ABC&ie=utf8&_sug_=n&_sug_type_=&page=BCA";
		String url = this.engineSite.getStartUrl();
		ArrayList<Article> list = new ArrayList<Article>();
		try {
     
			// 替换关键字
			url = url.replaceAll("ABC", URLEncoder.encode(keyword, "utf-8"));
			// 替换页数
			url = url.replace("CBA", pageorder + "");
			// jsoup获取doc对象
			Document doc = Jsoup.parse(new URL(url), 10000);
			// 必要时，重启cookie
			if (doc.getElementsByTag("ul").isEmpty()) {
     
				// 通过HttpClient得到Cookies
				HttpClient http = null;
				CookieStore httpCookieStore = new BasicCookieStore();
				http = HttpClientBuilder.create().setDefaultCookieStore(httpCookieStore).build();
				HttpGet httpRequest = new HttpGet("https://v.sogou.com/v?ie=utf8&query=&p=40030600");
				@SuppressWarnings("unused")
				HttpResponse httpResponse = null;
				try {
     
					httpResponse = http.execute(httpRequest);
				} catch (Exception e) {
     
					logger.error("HttpClient Connect Error :" + e);
				}
				List<Cookie> cookies = httpCookieStore.getCookies();
				
				// 设置cookie并发送请求
				HashMap<String, String> cookiesMap = new HashMap<String, String>();
				for (Cookie c : cookies) {
     
					cookiesMap.put(c.getName(), c.getValue());
				}

				doc = Jsoup.connect(url).cookies(cookiesMap).get();
			}

			// 根据li标签获取具体内容
			Elements Lis = doc.getElementsByTag("ul").get(1).getElementsByTag("li");

			// 循环生成文章对象
			for (int i = 0; i < Lis.size(); i++) {
     
				Article article = new Article();
				Element e = Lis.get(i);
				// 设置文章网站
				IWebsite web = new IWebsite();
				web.setWebUrl("https://weixin.sogou.com/");
				web.setWebName("sogouWeixin");
				article.setWebsite(web);
				// 设置url、author、content等
				article.setUrl(web.getWebUrl() + e.getElementsByAttribute("href").first().attr("href"));
				article.setAuthor(e.getElementById("sogou_vr_11002601_account_" + i).text());
				article.setAuthorUrl(web.getWebUrl() + e.getElementById("sogou_vr_11002601_account_" + i).attr("href"));
				// 设置content为摘要
				article.setContent(article.getDigest());
				// 设置发布时间
				String dataTime = getWeixinTime(e.selectFirst("div.s-p").attr("t").toString());
				article.setPostTime(dataTime);
				// 设置标题
				article.setEngineName("sogouWeixin");
				article.setSubject(doc.getElementsByTag("h3").get(i).text());
				// 设置唯一标识
				article.setUnicodeString(article.getUrl());

				list.add(article);
			}
		} catch (

		Exception e) {
     
			logger.error("Parse page Error :" + e);
		}
		return list;

上文代码中关键部分

效果

爬取上千行未出现验证码

最后

解决方案来自 https://blog.csdn.net/weixin_41186451/article/details/96980556
本帖仅供学习交流使用，禁止数据商用

不忘初心，技术改变世界