appium+mitmdump+夜神模拟器抓取抖音app

最近项目在搞抖音app，记录一下过程和遇到的坑

第一步 准备环境

1，appium
2，fiddle
3，夜神模拟器
4，Android SDK
详情可以借鉴这篇文章
https://www.cnblogs.com/qieyu/p/10450739.html

其中如果出现 adb devices 连接不到的情况就将上述文章中的替换adb.exe步骤更改为sdk的adb.exe复制到模拟器下bin目录下的adb.exe，然后将nox_adb.exe复制到桌面然后重命名为nox_adb.exe.bak，再将重命名后的文件放到模拟器bin目录下
再执行adb connect connect 127.0.0.1:62001

整个操作过程可以参考https://www.cnblogs.com/CYHISTW/p/11626990.html

第二步 编写操作app脚本

这个tap.py脚本是点击搜索框输入罗志祥然后点击用户进入用户主页

from appium import webdriver
# WebDriverWait用来加入时间判断，有时候控件元素需要过一段时间才会出现
from selenium.webdriver.support.ui import WebDriverWait
import time

'''
此版本适用于抖音6.0.0版本
'''
# 配置信息
option = {
    "platformName": "Android",
    "platformVersion": "5.1.1",
    "deviceName": "127.0.0.1:62001",
    # 自动化测试包名
    "appPackage": "com.ss.android.ugc.aweme",
    # 自动化测试Activity
    "appActivity": "com.ss.android.ugc.aweme.main.MainActivity",
    'automationName': 'uiautomator2',
    # 再次启动不需要再次安装
    "noReset": True,
    # unicode键盘 我们可以输入中文
    "unicodekeyboard": True,
    # 操作之后还原回原先的输入法
    "resetkeyboard": True
}

# 其中的4723就是appium服务启动时的端口号
driver = webdriver.Remote("http://localhost:4723/wd/hub", option)


# 得到窗口大小
def get_size():
    x = driver.get_window_size()['width']
    y = driver.get_window_size()['height']
    return (x, y)


# 放大镜按钮
try:
    # 使用resource-id查找按钮
    if WebDriverWait(driver, 5).until(lambda x: x.find_element_by_id('com.ss.android.ugc.aweme:id/aqz')):
        # 点击按钮
        driver.find_element_by_id('com.ss.android.ugc.aweme:id/aqz').click()
except:
    pass

WebDriverWait(driver, 5)
# 定位搜索框
if WebDriverWait(driver, 5).until(lambda x: x.find_element_by_id('com.ss.android.ugc.aweme:id/agq')):
    driver.find_element_by_id('com.ss.android.ugc.aweme:id/agq').click()
    # time.sleep(1)
    driver.find_element_by_id('com.ss.android.ugc.aweme:id/agq').send_keys('罗志祥')

# 点击搜索
driver.find_element_by_id('com.ss.android.ugc.aweme:id/agt').click()
WebDriverWait(driver, 5)
# 点击用户
try:
    driver.find_element_by_id('android:id/text1').click()
except:
    driver.tap([(299, 173), (359, 214)], 500)
time.sleep(4)
try:
    driver.find_element_by_id('com.ss.android.ugc.aweme:id/ab8').click()
except:
    driver.tap([(32, 279), (144, 391)], 500)
time.sleep(5)

上述脚本中用模拟器搜索用户的时候可能会返回找不到用户，可能是因为安装的Xposed Installer 没有激活

这个swipe.py脚本是自动上滑左滑进入用户主页

from appium import webdriver
# WebDriverWait用来加入时间判断，有时候控件元素需要过一段时间才会出现
from selenium.webdriver.support.ui import WebDriverWait
import time

# 配置信息
option = {
    "platformName": "Android",
    "platformVersion": "5.1.1",
    "deviceName": "127.0.0.1:62001",
    # 自动化测试包名
    "appPackage": "com.ss.android.ugc.aweme",
    # 自动化测试Activity
    "appActivity": "com.ss.android.ugc.aweme.main.MainActivity",
    # 再次启动不需要再次安装
    "noReset": True,
    # unicode键盘 我们可以输入中文
    "unicodekeyboard": True,
    # 操作之后还原回原先的输入法
    "resetkeyboard": True
}

# 其中的4723就是appium服务启动时的端口号
driver = webdriver.Remote("http://localhost:4723/wd/hub", option)


# # 得到窗口大小
def get_size():
    x = driver.get_window_size()['width']
    y = driver.get_window_size()['height']
    return (x, y)


def swipe_left():
    size = get_size()
    driver.swipe(int(size[0] * 0.85), int(size[1] * 0.5), int(size[0] * 0.15), int(size[1] * 0.5), 1000)
    print('向左滑动')


def swipe_up():
    size = get_size()
    driver.swipe(int(size[0] * 0.5), int(size[1] * 0.85), int(size[0] * 0.5), int(size[1] * 0.15), 1000)
    print('向上滑动')


def swipe_right():
    size = get_size()
    driver.swipe(int(size[0] * 0.15), int(size[1] * 0.5), int(size[0] * 0.85), int(size[1] * 0.5), 1000)
    print('向右滑动')


while True:
    WebDriverWait(driver, 100)
    # 测出等待四秒刚好是抖音加载时间
    time.sleep(4)
    # 模拟滑动
    swipe_left()
    swipe_left()
    time.sleep(0.2)
    swipe_right()
    time.sleep(0.2)
    swipe_up()
    time.sleep(0.2)
    swipe_left()
    time.sleep(0.2)
    swipe_right()
    time.sleep(0.2)
    swipe_up()

其中获取appPackage和appActivity获取方法参考下面这位大兄弟的文章
https://blog.csdn.net/qq_34374753/article/details/81605374

下面mitmdump_get_user_id.py 是mitmdump运行文件，获取到接口数据

import json
import re
from get_shop_detail import get_shop_detail


# from get_user_detail import handle_douyin_info


# 获取的数字都是详细数字，返回抖音展示数据
def show_num(st):
    if 100000000 > st > 10000:
        st = str(st)[:-4] + '.' + str(st)[-4] + 'w'
        return st
    elif st > 100000000:
        if int(str(st)[-8]) > 8:
            st = str(int(str(st)[:-8]) + 1) + '.' + '0' + '亿'
        else:
            st = str(st)[:-8] + '.' + str(st)[-8] + '亿'
        return st
    else:
        return st


# 'aweme.snssdk.com/aweme/v1/discover/search'这是搜索框内的接口
# aweme-hl.snssdk.com/aweme/v1/user/follower/list
# 函数名必须这样写 这是mitmdump规则
def response(flow):
    uid_list = []
    if 'https://aweme-hl.snssdk.com/aweme/v1/story/?' in flow.request.url:
        # print('输出连接story', flow.response.text)
        story_data = json.loads(flow.response.text)
        for u in story_data['story_feed']:
            s_uid = u['user_info']['uid']
            uid_list.append(s_uid)
    elif 'aweme-hl.snssdk.com/aweme/v1/aweme/post/?' in flow.request.url:
        # print('输出连接post', flow.response.text)
        post_data = json.loads(flow.response.text)
        p_uid = post_data['aweme_list'][0]['author']['uid']
        uid_list.append(p_uid)
    elif 'aweme-eagle-hl.snssdk.com/aweme/v1/user/?user_id' in flow.request.url:
        # print('输出连接user_id', flow.response.text)
        '''用户详情页，如果mitmdump没有获取到该链接则获取uid去请求网页版
        '''
        user_data = json.loads(flow.response.text)
        nickname = user_data['user']['nickname']
        # 粉丝数
        mplatform_followers_count = user_data['user']['mplatform_followers_count']
        # 认证
        custom_verify = user_data['user']['custom_verify']
        # 自我备注
        signature = user_data['user']['signature']
        # 关注
        following_count = user_data['user']['following_count']
        # 获赞数
        total_favorited = user_data['user']['total_favorited']
        s_uid = user_data['user']['uid']
        print(nickname, s_uid, mplatform_followers_count, signature, following_count, total_favorited, custom_verify)
        uid_list.append(s_uid)
    elif 'aweme-eagle-hl.snssdk.com/aweme/v1/feed/' in flow.request.url:
        # print('输出连接feed', flow.response.text)
        feed_data = json.loads(flow.response.text)
        for u in feed_data['aweme_list']:
            f_uid = u['author']['uid']
            uid_list.append(f_uid)
    elif 'aweme/v1/discover/search' in flow.request.url:
        user_data = json.loads(flow.response.text)
        for u_mes in user_data['user_list']:
            # 简称
            nickname = u_mes['user_info']['nickname']
            # 自我备注
            signature = u_mes['user_info']['signature']
            # 关注
            following_count = u_mes['user_info']['following_count']
            # 粉丝数
            follower_count = u_mes['user_info']['follower_count']
            # 获赞数
            total_favorited = u_mes['user_info']['total_favorited']
            # 抖音号
            unique_id = u_mes['user_info']['unique_id']
            # 认证
            custom_verify = u_mes['user_info']['custom_verify']
            # 头像图片
            cover = u_mes['user_info']['avatar_168x168']['url_list'][0]
            print(nickname,signature,follower_count,following_count,total_favorited,unique_id,custom_verify,cover)
    if len(set(uid_list)):
        for uuid in set(uid_list):
            for shop_info in get_shop_detail(uuid):
                if '该用户没有种草视频' in shop_info:
                    pass
                else:
                    # user_info = handle_douyin_info(uuid)
                    # print(user_info)
                    print(shop_info)

下面get_shop_detail.py是根据用户id判断是否有种草视频接口

import json

import requests


def get_num(num):
    if num[-2:] == '00':
        num = num[:-2]
    else:
        num = num[:-2] + '.' + num[-2]
    return num


def get_shop_detail(uid):
    url = f'https://aweme-hl.snssdk.com/aweme/v1/promotion/user/promotions/?count=2000&cursor=0&user_id={uid}&version=2&from=2&ts=1572857141&js_sdk_version=1.14.5.20&app_type=normal&os_api=22&device_type=google%20Pixel%202&device_platform=android&ssmix=a&iid=90888264944&manifest_version_code=600&dpi=320&uuid=865166010002072&version_code=600&app_name=aweme&version_name=6.0.0&openudid=00cfe04d074c9806&device_id=69289123246&resolution=900*1600&os_version=5.1.1&language=zh&device_brand=google&ac=wifi&update_version_code=6002&aid=1128&channel=wandoujia_aweme1&_rticket=1572857161583&mcc_mnc=46007&as=a1750e6b45437d154f4855&cp=ec37dd5c5af2b85ae1KcSg&mas=0144f35c4083bebfe452339d1c8b179f38acac1c2c662cac8c26ec'
    headers = {'accept-encoding': 'gzip, deflate, br', 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
               'cache-control': 'max-age=0',
               'cookie': 'odin_tt=15551ee734234ab8d70bd407a37fa44abb65f07804072751c61070fc46b0245df24e6ba1ab998e458b5850f6b1de4e7c5de439b885f2d14c7add4adc1141144c',
               'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'none', 'sec-fetch-user': '?1',
               'upgrade-insecure-requests': '1',
               'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}
    res = requests.get(url, headers=headers, verify=False)
    if res.status_code == 200:
        try:
            if len(res.json()['promotions']) != 0:
                for mes in res.json()['promotions']:
                    # 商品标题
                    shop_title = mes['title']
                    # 商品链接
                    shop_url = mes['url']
                    # 商品id
                    gid = mes['gid']
                    # 商品封面
                    elastic_img = mes['elastic_img'][0]['url_list'][0]
                    prices = str(mes['price'])
                    # 商品现价
                    price = get_num(prices)
                    market_prices = str(mes['market_price'])
                    # 商品原价
                    market_price = get_num(market_prices)
                    # 销售量
                    sales = mes['sales']
                    # 视频id
                    # last_aweme_id = mes['last_aweme_id']
                    # 优惠券值
                    if 'coupon' in json.loads(mes['extra']):
                        coupon_amount = json.loads(mes['extra'])['coupon']['coupon_amount']
                    else:
                        coupon_amount = '无'
                    # print(shop_title, gid, coupon_amount, elastic_img, price, shop_url, market_price, sales)
                    yield shop_title, gid, coupon_amount, elastic_img, price, shop_url, market_price, sales
            else:
                yield '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频', '该用户没有种草视频'
        except Exception as e:
            print('出错了', e)

运行步骤最好按照下面步骤来：
1，打开模拟器和appium-desktop
2，在命令窗口输入mitmdump -s mitmdump_get_user_id.py -p 8080
3，运行appium server
4，运行操作py脚本