Python爬虫之社区团购某团、某心、某多、某马微信小程序商品数据

   如有问题扫描微信咨询：airen789001

 

代码仅供学习交流，请勿用于非法用途 

主档商品名称

平台名称

平台商品名称

城市

佣金比例

平台售价

平台原价

成本价

商品规格

规格单位

售卖单位

标准价

标准到手价

开始时间

结束时间

怡宝饮用纯净水555ml

多多买菜

【21年5月生产】怡宝纯净水555ml*12瓶量贩装

合肥市

0

13.7

15.16

13.7

12

瓶

件

1.141667

1.141667

2021/6/23 0:00

2021/6/23 23:00

怡宝饮用纯净水555ml

橙心优选

怡宝 饮用纯净水 555ml*24瓶/箱

合肥市

8

27.5

48

25.3

24

瓶

件

1.145833

1.054167

2021/6/23 0:00

2021/6/23 23:00

怡宝饮用纯净水555ml

橙心优选

怡宝 纯净水 555ml*12瓶/提

合肥市

8

13.5

15.9

12.42

12

瓶

件

1.125

1.035

2021/6/23 0:00

2021/6/23 23:00

怡宝饮用纯净水555ml

多多买菜

3月份 怡宝纯净水555ml*24瓶

合肥市

0

23.9

28.9

23.9

24

瓶

件

0.995833

0.995833

2021/6/23 0:00

2021/6/23 23:00

怡宝饮用纯净水555ml

橙心优选

怡宝 饮用纯净水 555ml/瓶

合肥市

10

1.2

2.5

1.08

1

瓶

瓶

1.2

1.08

2021/6/23 0:00

2021/6/23 23:00

# -*- coding: utf-8 -*-
# !/usr/bin/env python
# @Author  : 好东西来分享
# @File    : 某团.py
import warnings
from spider import utils as ut
import datetime
from copy import deepcopy
import json
import time
from urllib.parse import quote
warnings.filterwarnings("ignore")

def GetCity(token):
    headers = {
        "t": token,
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat"
    }
    url = "https://grocery-gh.meituan.com/api/g/grouphead/gw/mini/exclusiveGroupChat"
    content = ut.parse_url(url=url,headers=headers,method="get")
    data = content.get("data")
    city_df = ut.get_city_info()
    if data is not None:
        name = data["cityName"]
        city_df = city_df.loc[city_df["city_name"].str.contains(name)]
        num = city_df.shape[0]
        if num == 1:
            return city_df
        else:
            return "=========城市没判断准确！========="
    else:
        return "=======获取城市信息失败！=========="


def extract_father_info(token):
    """获取大类信息"""
    # todo 这里可能需要修改
    headers = {
        "t": token,
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat"
    }
    url = "https://grocery-gh.meituan.com/api/g/grouphead/service/market/products/tabs"
    content = ut.parse_url(url=url, headers=headers, method="get")
    print(content)
    tabs = content.get("data").get("tabs")
    data_list = list()
    for i in tabs:
        text = i["text"]
        href = "https://grocery-gh.meituan.com" + i["href"]
        data_list.append((text,href))
    return data_list


def extract_data(token,url,item):
    res = deepcopy(item)
    headers = {
        "t": token,
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat"
    }
    i = 0
    while 1:
        print("开始获取【%s-%s】商品第【%s】页" % (res["app_name"], res["column"],i+1))
        if res["column"] == "今日推荐":
            newurl = url + "?pageSize=20&pageNo=%s&lastCachePosition=%s&sortType=0"%(i,20*i)
        elif res["column"] == "附近热卖":
            newurl = url + "?pageSize=20&pageNo=%s&scene=0&utm_medium"%i
        else:
            newurl = url + "?pageSize=20&pageNo=%s&utm_medium"%i

        content = ut.parse_url(url=newurl, headers=headers, method="get")
        data = content["data"]
        productList = data["productList"]
        if productList:
            # 有数据 转化存储
            insert_item(res, productList)
            # 翻页
            i += 1
        else:
            break
def insert_item(res,productList):
    item = deepcopy(res)
    data_list = list()

    for product in productList:
        print(product)
        item["app_sku_id"] = product["skuBaseId"]
        item["app_sku"] = product["skuName"]
        item["share_rate"] = format(float(product["rate"]),'.2f')
        item["sell_price"] = format(float(product["promotionPrice"]),'.2f')
        if product["originPrice"] == "":
            item["origin_price"] = 0.00
        else:
            item["origin_price"] = format(float(product["originPrice"]),'.2f')
        item["price"] = format(float(product["promotionPrice"]) * (1 - (float(product["rate"]) / 100)),'.2f')
        item["pic_url"] = product["picture"]
        item["limitcount"] = product["promotionLimitCount"]
        if "start_time" in item.keys():
            item["start_time"] = item["start_time"]
        else:
            item["start_time"] = item["update_date"] + " 00:00:00"
        item["stop_time"] = item["update_date"] + " 23:00:00"
        item["share_path"] = "pages/skuDetail/index.html?__RST=xxxxxxxxxxxxx&skuCardPicUrl=%s&need_preload=1&skuId=%s&originalPoiId=142932237183389" % (quote(product["picture"], "utf-8"), product["skuBaseId"])
        item["data_source"] = json.dumps(product, ensure_ascii=False)

        sql_data = eval('("""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""","""{}""",None,None,None,"""{}""", None,{},"""{}""","""{}""","""{}""","""{}""")'.format(
            str(item["app_id"]), item["app_name"], str(item["app_sku_id"]), item["app_sku"], item["column"],
            str(item["city_id"]),item["city_name"],item["update_date"],item["share_rate"], item["sell_price"],
            item["origin_price"],item["price"], item["pic_url"],
            int(item["limitcount"]), item["start_time"], item["stop_time"], item["share_path"], item["data_source"]))
        data_list.append(sql_data)
    ut.insert_data(data_list)


def second_kill(token, info_url, item):
    res = deepcopy(item)
    url = info_url + "?utm_term=2.91.1&sysName=Windows&sysVersion=10&business=gh&debug=false&utm_medium=wxapp&bizId=4&fulfillmentOrderGrayScale=false&batchPickupV0=false&orderListSearchOptGrayScale=true"
    headers = {
        "t": token,
        "mp-mtgsig":'{"a1":"1.0","a2":1621323513286,"a3":"589z9ywx28y65vw8y75u56835w5yxz238212xx310xx989884wzu4xx5","a4":"8ac7a91c334ad77d1ab78c7fcd4b50a998a67bc6dbf69400","a5":"sf1beiELhz+Xyg7Bjrt4WiBfaY5vpkGrCD+xaMHm/GCzgF68kRlDwseX15HeKmD427O8RPN/0//e0hrVLpmTQyIrFn4KfC7lgSnpz1mkANwQ/S+bsOushq0ovjgnpcc7ODCBkY6lOAMLRsx5ZfUQXRXhZLYjrVTEbmWCXbVzNuTvB/KN52FJ/l4oT9rj2aPoSrBeTjLl6JNCJ9bvGyObkkIiL1rFJ6fwNG5uOrr2i+xtucj8jhhjDRraXxuoZVot3quW8FjADvzE6nizxtyrr+KQgll01XLKbSWZM3OFYoI9eOQEeHSGKX69","a6":"w1.0wZD6pex7gKE7BSbNaRNg2mxDIZYZmtB0FRfg0HOH4zk1TGYNLhg3u4NCzb7d+/FIUF4/QGJAVn/svXkS0YDatsFqXQX7AqpLWEtmR3kq0zhwh12faHtXFe7OV79vZn1S4xSS6vPWFQy6tM9gfoG71uAC2r2Y5Z8RiTssEFYvtU3CK8+Js5FxQiO/Ukdu3aqNerp0kItkY14/5EqOTdPwoNDepRAMvzrPCgHuEtOTlIMkWsugTC+DZmDQPoUkETBkopzxIR6eRnA0dg+NKNxNAPHefxOtK7zZyzhOwG7MW7OsrmKZonoLjdy8+f8Ka71eIl1LYLUHibVssomM4TulG1vG0SXEHezm5sQZ5RcsA2DN3IMia21gAT96owpRI48tzvWQadsr+1B66xgBRlAbosGuBLWiSaCbUkCKkkxb8B+aWerlHsHVReGCtz2tHt9aHpx0fO4eEHGN3Nk1sF6ak9PWhk/rzHi61Sav59KUFl4=","a7":"wx4474ed752dbe0955"}',
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat"
    }
    content = ut.parse_url(url=url, headers=headers, method="get")

    data_list = content["data"]
    for i in data_list:
        productList = i["productList"]
        startTime = i["startTime"]/1000
        timeValue = time.localtime(startTime)
        res["start_time"] = time.strftime("%Y-%m-%d %H:%M:%S", timeValue)
        insert_item(res, productList)
    print("【%s-%s】插入成功"%(res["app_name"], res["column"]))

def main():
    # todo 可能需要修改的
    token = "xxxxxxxxx"
    # 1 数据详情
    item = dict()
    # # 2 日期和app
   
    item["update_date"] = str(datetime.datetime.today())[0:10]
    item["app_id"] = 101
    item["app_name"] = "某团优选"
    # # 3 城市
    city_df = GetCity(token)
    item["city_id"] = city_df["city_id"].values[0]
    item["city_name"] = city_df["city_name"].values[0]
    # 分类信息
    father_list = extract_father_info(token)

    if father_list:
        for i in father_list:
            item["column"],info_url = i

            if item["column"] == "限时秒杀":
                # 秒杀单独提出来
                second_kill(token, info_url, item)
            elif item["column"] == "拉新有奖":
                continue
            else:
                extract_data(token, info_url, item)

    else:
        print("========大类信息获取失败，应该需要调整参数")


    print("====================【%s %s】商品信息获取完毕"%(item["update_date"],item["app_name"]))


if __name__ == '__main__':
    main()