阅读更多
def self.sang_carwler
agent = Mechanize.new
cc1 = ConsumableCategory.find_or_create_by(name: "生命科学", parent_id: 0)
resp = agent.get("https://www.XXX.com").search("li.subnav_cat dl.sub_dl")
resp.each do |e1|
catalog_name2 = e1.search(".sub_dt a").first.children.first.text.strip
next unless ["生化试剂", "分子生物学", "蛋白质科学", "抗体", "细胞生物学"].include?(catalog_name2)
cc2 = ConsumableCategory.find_or_create_by(name: catalog_name2, parent_id: cc1.id)
e1.search(".sub_dd a").each do |e2|
catalog_name3 = e2.children.first.text.strip
cc3 = ConsumableCategory.find_or_create_by(name: catalog_name3, parent_id: cc2.id)
end
end
end
－－－－－－－－－－－－－－
# 基于接口的查询
def self.guoyao_crawler(titles = [])
menu_resp = HTTParty.get "https://XXX/reagent-front/indexApi/goodsClassList"
menu_resp["data"].each do |catalog_1|
next unless titles.include?(catalog_1["gcName"])
cc1 = ConsumableCategory.find_or_create_by(name: catalog_1["gcName"], parent_id: 0)
catalog_1["classList"].each do |catalog_2|
next if catalog_1["gcName"] == "化学试剂" && ["通用试剂", "高纯试剂", "色谱应用", "分析标准品"].include?(catalog_2["gcName"])
cc2 = ConsumableCategory.find_or_create_by(name: catalog_2["gcName"], parent_id: cc1.id)
catalog_2["classList"].each do |catalog_3|
cc3 = ConsumableCategory.find_or_create_by(name: catalog_3["gcName"], parent_id: cc2.id)
if catalog_1["gcName"] == "化学试剂"
page_size = 100
page_no = 1
flag = true
while flag do
chemical_resp = HTTParty.get "https://XXX/reagent-front/goodsApi/getGoodsList?pageSize=#{page_size}&pageNo=#{page_no}&searchType=gcIdSearch&keyword=#{catalog_3['gcId']}"
if chemical_resp["data"].first["pageCount"].to_i > 0
chemical_resp["data"].first["listApiGoods"].each do |goods|
MenuChemical.find_or_create_by(consumable_category_id: cc3.id, cas: goods["casIndexNo"])
end
page_no += 1
else
flag = false
end
end
end
end
end
end
end
－－－－－－－－－－－－－－
def self.worm(sleep_second = 0.15)
agent = Mechanize.new
['nav-1', 'nav-2', 'nav-3', 'nav-4'].each do |nav|
agent.get("http://www.xxxx/zh_cn/").search("li.level0.#{nav} ul li ul li a").each do |link1|
puts "#{link1.attributes['href'].value} #{link1.children.children.text}"
fenlei = link1.children.children.text
product_list = agent.get(link1.attributes['href'].value)
total_count = product_list.search(".toolbar-number").last&.children&.text.to_i
total_page = total_count % 15 == 0 ? (total_count / 15) : ((total_count / 15) + 1)
(1..total_page).to_a.each do |page|
product_list = agent.get("#{link1.attributes['href'].value}?p=#{page}")
break if product_list.search('div.actions-primary a').blank?
product_list.search('div.actions-primary a').each do |product_link|
sleep sleep_second
product_page = agent.get(product_link.attributes['href'].value)
product_no = product_link.attributes['href'].value.gsub('http://www.xxx/zh_cn/','').gsub('.html', '').upcase
cas = product_page.search("#product_addtocart_form > div.product-shop > div.product-info > span:nth-child(2) > a").children.text
if product_page.search("#super-product-table thead tr").children.select{|c| c.name == 'th'}.map{|th| th.children.text.to_s.strip} == ["货号", "规格", "库存", "价格", "数量"]
product_page.search("#super-product-table tbody tr").each do |tr|
tds = tr.children.select{|c| c.name == 'td'}
package_unit = tds[0].children.text.to_s.strip.gsub("#{product_no}-",'')
package = package_unit.to_f
unit = package_unit.slice(/[a-zA-Z]+/).downcase
purity = tds[1].children.text.to_s.strip
stock = tds[2].children.text.to_s.strip
ajax_price_id = tds[3].attributes['attr'].value
price = 0
response = HTTParty.post("http://www.xxx/zh_cn/catalogb/ajax/price", body: {ajax_price_id => ajax_price_id})
price = Nokogiri::HTML(JSON.parse(response.parsed_response)[ajax_price_id]).search("p span.price").last.text.gsub(/[^0-9]/,'').to_f / 100 if response&.parsed_response.present?
ReagentCategory.create(name: 'ald', fenlei: fenlei, product_no: product_no, cas: cas, package: package, unit: unit, stock: stock, price: price, purity: purity, ajax_price_id: ajax_price_id, vendor_id: VENDOR_ID, company_id: COMPANY_ID)
end
end
end
end
end
end
end
