app爬取小知识点

aapt 获取到appPackage和appActivity

appium+夜神模拟器+python安卓app爬虫初体验

adb获取到appPackage和appActivity

1.adb shell 进入模拟器

 2.logcat | grep cmp=

3.去模拟器打开相应的软件

4.查看日志，以斜线分割

文字反扒

python 爬虫之字体(@font-face)防爬

xponsed框架 ,justtruestme组件 安装（对于真机数据抓取app无法联网）

xponsed下载地址，justtruestme下载地址,过程简单可以参考http://www.ttk7.cn/post-99.html

Android7.1.2对应版本3.15 http://bbs.xyaz.cn/thread-9717-1-1.html

5.1.1 对应版本3.12 https://www.qqtn.com/azsoft/268541.html

mitmdump监听

App爬虫神器mitmproxy和mitmdump的使用

mitmdump -s douyin.py -p 8888

指定douyin.py文件 多端口在8888

douyin.py 如下

import json
def response(flow):
    if "aweme/v1/user/follower/list" in flow.request.url:
        for user in json.loads(flow.response.text)['followers']:
            douyin_info={}
            douyin_info['share_id']=user['uid']
            douyin_info['douyin_id']=user['short_id']
            douyin_info['nickname']=user['nickname']
            print(douyin_info)

成功输出

selenium 被服务器检测，解决方法

模拟用户弧线滑动

import random
import math
import mouse



def distance(location1, location2):
    return math.sqrt((location1[0] - location2[0])**2 + (location1[1] - location2[1])**2)

def getCircleXY(a, x0, y0, r):
    x = x0 + r * math.cos(a)
    y = y0 + r * math.sin(a)
    return (x,y)

def getAngleXY(x, y, x0, y0):
    return math.atan2(y-y0, x-x0)

def getAngle(location1, location0):
    return getAngleXY(location1[0], location1[1], location0[0], location0[1])

def dragDropX(location2, dragTime,angleA=math.pi/6,location1=mouse.get_position()):
    # print ("[Debug]start dragDropX function")
    x1 = location1[0]
    y1 = location1[1]
    x2 = location2[0]
    y2 = location2[1]
    connerA = angleA
    connerB = (math.pi - connerA)/2
    d0 = math.sqrt((x2-x1)**2 + (y2-y1)**2)
    r = d0 * math.sin(connerB)/math.sin(connerA)
    connerC = math.atan2((y2-y1),(x2-x1))
    connerD = connerC + connerB

    x0 = x1 + r * math.cos(connerD)
    y0 = y1 + r * math.sin(connerD)
    location0 = (x0, y0)

    startPoint = location1
    endPoint = location2
    startAngle = getAngle(startPoint, location0)
    endAngle = getAngle(endPoint, location0)
    if abs(endAngle - startAngle) > math.pi:
        if endAngle < startAngle:
            endAngle += math.pi * 2
        else:
            startAngle += math.pi * 2
    n = 30
    jitter = math.ceil(r * abs(endAngle - startAngle)/n/10)
    # print(startPoint)
    mouse.move(startPoint[0],startPoint[1],True,0)
    angleStep = (endAngle - startAngle) / n
    for i in range(n):
        angle = startAngle + angleStep * i
        lo = getCircleXY(angle, x0, y0, r)
        # print((lo[0]+random.randint(-jitter,jitter), lo[1]+random.randint(-jitter,jitter)))
        dragtimeOne=dragTime*4 if i >= 29/30 * n else (dragTime*2 if i >= 28/30*n else dragTime)
        mouse.move(lo[0]+random.randint(-jitter,jitter),  lo[1]+random.randint(-jitter,jitter), True, dragtimeOne)
    # print(endPoint)
    for i in range(random.randint(1, 3)):
        mouse.move(random.uniform(-3,3), random.uniform(-3,3), False, 0.05)
    mouse.move(endPoint[0], endPoint[1], True, 0.1)

if __name__=="__main__":

    print(mouse.get_position())
    # location1=(10,10)
    # location2=(200,500)
    # dragDropX(location2,0.01,math.pi/3)
    dragDropX((80, 50), 0.01, angleA=math.pi/90,
              location1=(500, 800))

js脚本识别到特征“$cdc_asdjflasutopfhvcZLmcfl_” 修改chromedriver