使用爬虫技巧对有道词典网站进行操作

有道词典的网站的措施做的还是不错的，今天我们的任务就是把它的功能拿到pycharm里面来实现

首先  我们导入requests模块

接下来进行接口分析

找到真的接口url

添加a

添加m

但是直接请求却不能成功

所以，应该是在某个地方上有所不同，才会出现这样的结果，所以重新进行分析

根据经验，像这种表单数据应该出自一人之手，所以我们可以在js文件中找类似的数据

找到适合的进行格式化

查找我们之前发现的不同的关键词

接下来就是对向值进行相似的处理

'=========================='
r = str(int(time.time()*1000))
i = r + str(int(random.random()*10))
# print(i)
par1 = "fanyideskweb" + keyword + i + "@6f#X3=cCuncYssPsuRUE"
md5 = hashlib.md5()
md5.update(bytes(par1,encoding='utf_8'))
sign = md5.hexdigest()
# print(sign)
'=========================='

最后整合代码

import requests
import time
import hashlib
import random
url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'



keyword = input('>>>')

'=========================='
r = str(int(time.time()*1000))
i = r + str(int(random.random()*10))
# print(i)
par1 = "fanyideskweb" + keyword + i + "@6f#X3=cCuncYssPsuRUE"
md5 = hashlib.md5()
md5.update(bytes(par1,encoding='utf_8'))
sign = md5.hexdigest()
# print(sign)
'=========================='



headers = {
    'Accept':'application/json, text/javascript, */*; q=0.01',
    # Accept-Encoding:gzip, deflate
    'Accept-Language':'zh-CN,zh;q=0.9',
    'Connection':'keep-alive',
    'Content-Length':str(233+len(keyword)),
    'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
    'Cookie':'OUTFOX_SEARCH_USER_ID_NCOO=1325182775.0226092; OUTFOX_SEARCH_USER_ID="[email protected]"; _ga=GA1.2.513014409.1556033509; _gid=GA1.2.176139140.1556033509; JSESSIONID=aaalA8Bujl98-5-yjrxPw; ___rl__test__cookies=1556242664075',
    'Host':'fanyi.youdao.com',
    'Origin':'http://fanyi.youdao.com',
    'Referer':'http://fanyi.youdao.com/',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    'X-Requested-With':'XMLHttpRequest',
}



data_dic={
    'i':keyword,
    'from':'AUTO',
    'to':'AUTO',
    'smartresult':'dict',
    'client':'fanyideskweb',
    'salt':i,
    'sign':sign,
    'ts':r,
    'bv':'ab57a166e6a56368c9f95952de6192b5',
    'doctype':'json',
    'version':'2.1',
    'keyfrom':'fanyi.web',
    'action':'FY_BY_REALTlME',
}

response = requests.post(url=url,data=data_dic,headers=headers)
data = response.json()
print(data)

整合代码

import hashlib


def getMd5(value):
    md5 = hashlib.md5()
    md5.update(bytes(value, encoding='Utf-8'))
    md5_str = md5.hexdigest()
    return md5_str

if __name__ == '__main__':
    #输入单词
    keyword = input('翻译的单词：')

    #生成salt的值
    r = str(int(time.time()*1000))
    salt = r + str(int(random.random()*10))

    #生成sign
    value = "fanyideskweb" + keyword + salt + "@6f#X3=cCuncYssPsuRUE"
    md5_str = getMd5(value)

    #生成header
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Language': 'zh-CN,zh;q=0.9,und;q=0.8,en;q=0.7',
        'Connection': 'keep-alive',
        'Content-Length': str(233+len(keyword)),
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Cookie': '[email protected]; OUTFOX_SEARCH_USER_ID_NCOO=6241167.578427844; _ga=GA1.2.1519057173.1535511777; JSESSIONID=aaamFTGK4qLCf4uKcrxPw; ___rl__test__cookies=1556242390777',
        'Host': 'fanyi.youdao.com',
        'Origin': 'http://fanyi.youdao.com',
        'Referer': 'http://fanyi.youdao.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest'
    }

    #生成表单数据
    data_dic = {
        'i': keyword,
        'from': 'AUTO',
        'to': 'AUTO',
        'smartresult': 'dict',
        'client': 'fanyideskweb',
        'salt': salt,
        'sign': md5_str,
        'ts': r,
        'bv': 'ae62d2e2541901f6ebf99ec18e429e3f',
        'doctype': 'json',
        'version': '2.1',
        'keyfrom': 'fanyi.web',
        'action': 'FY_BY_REALTlME'
    }

    #接口地址
    url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'
    #发起网络请求
    response = requests.post(url=url,data=data_dic,headers=headers)
    print(response.text)