WebScraper爬虫模板

想学WebScraper戳下面博客的教材，非常全面

• 卤代烃实验室

知乎

• 首页搜索 只抓提问

{"_id":"zhihu","startUrl":["https://www.zhihu.com/search?q=%E6%99%BA%E8%83%BD%E8%A7%86%E9%A2%91%E9%97%A8%E9%93%83&utm_content=search_suggestion&type=content"],"selectors":[{"id":"container","type":"SelectorElementScroll","parentSelectors":["_root"],"selector":"div[data-za-detail-view-path-module='AnswerItem']:nth-of-type(n+7) div.List-item:nth-of-type(-n+150)","multiple":true,"delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["container"],"selector":"a","multiple":false,"delay":0},{"id":"关注者","type":"SelectorText","parentSelectors":["link"],"selector":"button strong","multiple":false,"regex":"","delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["link"],"selector":"div.Card:nth-of-type(1) a.QuestionMainAction","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["link"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0}]}

• 等你来答 - 人气问题

{"_id":"wait_one","startUrl":["https://www.zhihu.com/question/waiting"],"selectors":[{"id":"container","type":"SelectorElementScroll","parentSelectors":["_root"],"selector":"div.ListQuestionItem:nth-of-type(-n+100)","multiple":true,"delay":0},{"id":"answer_detail","type":"SelectorLink","parentSelectors":["container"],"selector":"a.ListQuestionItem-title","multiple":false,"delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["answer_detail"],"selector":".List-headerText span","multiple":false,"regex":"","delay":0},{"id":"关注者","type":"SelectorText","parentSelectors":["answer_detail"],"selector":".Button strong","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["answer_detail"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0}]}

• 等你来答 - 潜力好问

{"_id":"wait_two","startUrl":["https://www.zhihu.com/question/waiting?type=potential"],"selectors":[{"id":"cotainer","type":"SelectorElementScroll","parentSelectors":["_root"],"selector":"div.ListQuestionItem:nth-of-type(-n+100)","multiple":true,"delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["cotainer"],"selector":"a.ListQuestionItem-title","multiple":false,"delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["link"],"selector":".List-headerText span","multiple":false,"regex":"","delay":0},{"id":"关注者","type":"SelectorText","parentSelectors":["link"],"selector":".Button strong","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["link"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0}]}

• 等你来答 - 人人答

{"_id":"wait_four","startUrl":["https://www.zhihu.com/question/waiting?type=everyone"],"selectors":[{"id":"cotainer","type":"SelectorElementScroll","parentSelectors":["_root"],"selector":"div.ListQuestionItem:nth-of-type(-n+100)","multiple":true,"delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["cotainer"],"selector":"a.ListQuestionItem-title","multiple":false,"delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["link"],"selector":".List-headerText span","multiple":false,"regex":"","delay":0},{"id":"关注数","type":"SelectorText","parentSelectors":["link"],"selector":".Button strong","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["link"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0}]}

• 等待回答

{"_id":"zhihu_wait","startUrl":["https://www.zhihu.com/topic/19903720/unanswered"],"selectors":[{"id":"scroll","type":"SelectorElementScroll","parentSelectors":["_root"],"selector":"div.List-item","multiple":true,"delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["scroll"],"selector":".QuestionItem-title a","multiple":false,"delay":0},{"id":"关注者","type":"SelectorText","parentSelectors":["link"],"selector":".Button strong","multiple":false,"regex":"","delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["link"],"selector":".List-headerText span","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["link"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0},{"id":"第一个回答点赞数","type":"SelectorText","parentSelectors":["link"],"selector":"[itemprop='acceptedAnswer'] .Voters button","multiple":false,"regex":"","delay":0},{"id":"第二个回答点赞数","type":"SelectorText","parentSelectors":["link"],"selector":"[data-za-index='1'] .Voters button","multiple":false,"regex":"","delay":0}]}

知乎好物推荐 选蓝海题逻辑

• 选品
• 选品关键词问题 5个
• 抓取关键词问题下前20个回答
• 百度上跟知乎同步

知乎搜索问题后第一页回答

{"_id":"zhihu","startUrl":["https://www.zhihu.com/search?q=%E6%89%AB%E5%9C%B0%E6%9C%BA%E5%99%A8%E4%BA%BA%E6%9C%89%E7%94%A8%E5%90%97&utm_content=search_suggestion&type=content","https://www.zhihu.com/search?q=%E6%89%AB%E5%9C%B0%E6%9C%BA%E5%99%A8%E4%BA%BA%E5%92%8C%E5%90%B8%E5%B0%98%E5%99%A8&utm_content=search_suggestion&type=content","https://www.zhihu.com/search?q=%E6%89%AB%E5%9C%B0%E6%9C%BA%E5%99%A8%E4%BA%BA%E6%B5%8B%E8%AF%84&utm_content=search_suggestion&type=content","https://www.zhihu.com/search?q=%E6%89%AB%E5%9C%B0%E6%9C%BA%E5%99%A8%E4%BA%BA%E4%BB%80%E4%B9%88%E7%89%8C%E5%AD%90%E5%A5%BD&utm_content=search_suggestion&type=content","https://www.zhihu.com/search?q=%E6%89%AB%E5%9C%B0%E6%9C%BA%E5%99%A8%E4%BA%BA%E6%8E%A8%E8%8D%90&utm_content=search_suggestion&type=content"],"selectors":[{"id":"container","type":"SelectorElement","parentSelectors":["_root"],"selector":"[data-za-detail-view-path-module='AnswerItem'] div.List-item","multiple":true,"delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["container"],"selector":"a","multiple":false,"delay":0},{"id":"关注者","type":"SelectorText","parentSelectors":["link"],"selector":"button strong","multiple":false,"regex":"","delay":0},{"id":"回答数","type":"SelectorText","parentSelectors":["link"],"selector":"div.Card:nth-of-type(1) a.QuestionMainAction","multiple":false,"regex":"","delay":0},{"id":"浏览数","type":"SelectorText","parentSelectors":["link"],"selector":"div.NumberBoard-item strong","multiple":false,"regex":"","delay":0}]}

百度搜索问题后前20个回答