项目背景：现阶段我们项目主要有两大场景，一是交易风控，二是账户风控，两大的场景的很多规则都和设备ID有关，比如设备黑名单，设备A在黑名单库并且相关规则开启，设备A请求交易时就会有预警事件发生，所以设备ID的生成逻辑至关重要，主要和A、B、C 三大因素有关，大概如下：
1、同A，不管后面的B,C是否不一致
首先根据传入消息的A ，到ES中查询 如果存在相同的则用原A的设备ID(DeviceID)
2、不同A
1）B一样，C不一样，则用原B的设备ID(DeviceID)
2）C一样,B不一样，则用原C的设备ID(DeviceID)
3）B不一样，C不一样，则生成新设备ID(DeviceID)，生成新的设备ID的逻是原来的最大的设备ID号，加1最为新的设备ID，例如：原数据中最大设备ID是D000000009，则新的设备ID是D000000010.
设备ID的生成在数据量大并发的情况下是非常有可能是同一时间过来的，所以，当开启脚本后，即使是A、B、C因子都不一样的时候，因为生成时间是相同的（精确到毫秒级），所以生成的设备ID还是相同的（实际情况下是不可能生成相同设备的），此问题的解决方案有很多，可以搜索，分布式ID的生成，我们开发采取的方案是使用redis的原子性，ID生成器解决的。主要的locust脚本如下：
#!/user/bin/env python
#coding=utf-8
author = 'zengweifang'
#风控系统的压测脚本
#第一步：模拟设备指纹的100万数据的生成
from locust import HttpLocust,TaskSet,task
import queue
import json
import requests
from random import Random
import random
import time
import string
import socket
import struct
RockMQ = "http://192.168.46.154:8765/topic/sendTopicMessage.do"
def AutoGeneratedString(number):
'''随机生成字符串方法，主要用于输入框不能超过多少字符串的场景,此一次性产生的最大的字符串是62个'''
return ''.join(random.sample(string.ascii_letters + string.digits, number))
def getRandomBool():
'''随机获取是否是代理'''
isProxy = ["true","false"]
return random.sample(isProxy, 1)[0]
def get_random_ip():
'''随机生成合法的IP'''
RANDOM_IP_POOL=['192.168.10.222/0']
str_ip = RANDOM_IP_POOL[random.randint(0,len(RANDOM_IP_POOL) - 1)]
str_ip_addr = str_ip.split('/')[0]
str_ip_mask = str_ip.split('/')[1]
ip_addr = struct.unpack('>I',socket.inet_aton(str_ip_addr))[0]
mask = 0x0
for i in range(31, 31 - int(str_ip_mask), -1):
mask = mask | ( 1 << i)
ip_addr_min = ip_addr & (mask & 0xffffffff)
ip_addr_max = ip_addr | (~mask & 0xffffffff)
return socket.inet_ntoa(struct.pack('>I', random.randint(ip_addr_min, ip_addr_max)))
def getUnixTime():
'''获得13位的unix时间戳:1519800472673'''
return int(round(time.time() * 1000))
def getPCUserAgent():
'''获取PC端的UserAgent'''
pc_user_agent = [
'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1',
'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',
'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',
'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',
'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)']
list(set(pc_user_agent))
return random.sample(pc_user_agent, 1)[0]
def messageDict(fingerId, sessionId):
topic = {"topic": "meta-fingerprint"}
tag = {"tag": ""}
key = {"key": ""}
appId = {"appId": 'A10007'}
clientType = {"clientType": "1"}
fingerId = {"fingerId": fingerId}
ip = {"ip": get_random_ip()}
proxyFlag = {"proxyFlag": getRandomBool()}
sessionId = {"sessionId": sessionId}
systemId = {"systemId": '7b6a99f3bce14915863cde5104bdf2c3'}
timestamp = {"timestamp": str(getUnixTime())}
user_agent = '{\"key\":\"user_agent\",\"value\":\"%s\"}' % getPCUserAgent()
language = '{\"key\":\"language\",\"value\":\"zh-cn\"}'
color_depth = '{\"key\":\"color_depth\",\"value\":32}'
device_memory = '{\"key\":\"device_memory\",\"value\":-1}'
pixel_ratio = '{\"key\":\"pixel_ratio\",\"value\":\"\"}'
hardware_concurrency = '{\"key\":\"hardware_concurrency\",\"value\":\"unknown\"}'
resolution = '{\"key\":\"resolution\",\"value\":[1920,1080]}'
available_resolution = '{\"key\":\"available_resolution\",\"value\":[1920,1040]}'
timezone_offset = '{\"key\":\"timezone_offset\",\"value\":-480}'
session_storage = '{\"key\":\"session_storage\",\"value\":1}'
local_storage = '{\"key\":\"local_storage\",\"value\":1}'
add_behavior = '{\"key\":\"add_behavior\",\"value\":1}'
cpu_class = '{\"key\":\"cpu_class\",\"value\":\"x86\"}'
navigator_platform = '{\"key\":\"navigator_platform\",\"value\":\"Win32\"}'
do_not_track = '{\"key\":\"do_not_track\",\"value\":\"unknown\"}'
ie_plugins = '{\"key\":\"ie_plugins\",\"value\":[\"AcroPDF.PDF\",null,null,null,\"MacromediaFlashPaper.MacromediaFlashPaper\",\"Msxml2.DOMDocument\",\"Msxml2.XMLHTTP\",null,null,null,null,null,null,\"Scripting.Dictionary\",null,\"Shell.UIHelper\",\"ShockwaveFlash.ShockwaveFlash\",null,\"TDCCtl.TDCCtl\",\"WMPlayer.OCX\",null,null]}'
adblock = '{\"key\":\"adblock\",\"value\":false}'
has_lied_languages = '{\"key\":\"has_lied_languages\",\"value\":false}'
has_lied_resolution = '{\"key\":\"has_lied_resolution\",\"value\":false}'
has_lied_os = '{\"key\":\"has_lied_os\",\"value\":false}'
has_lied_browser = '{\"key\":\"has_lied_browser\",\"value\":false}'
touch_support = '{\"key\":\"touch_support\",\"value\":[0,false,false]}'
js_fonts = '{\"key\":\"js_fonts\",\"value\":[\"Arial\",\"Arial Black\",\"Calibri\",\"Cambria\",\"Cambria Math\",\"Comic Sans MS\",\"Consolas\",\"Courier\",\"Courier New\",\"Georgia\",\"Helvetica\",\"Impact\",\"Lucida Console\",\"Lucida Sans Unicode\",\"Microsoft Sans Serif\",\"MS Gothic\",\"MS PGothic\",\"MS Sans Serif\",\"MS Serif\",\"Palatino Linotype\",\"Segoe Print\",\"Segoe Script\",\"Segoe UI\",\"Segoe UI Light\",\"Segoe UI Semibold\",\"Segoe UI Symbol\",\"Tahoma\",\"Times\",\"Times New Roman\",\"Trebuchet MS\",\"Verdana\",\"Wingdings\"]}'
fingerParametersList = [user_agent, language, color_depth, device_memory, pixel_ratio, hardware_concurrency,
resolution, available_resolution, timezone_offset,
session_storage, local_storage, add_behavior, cpu_class, navigator_platform,
do_not_track, ie_plugins, adblock, has_lied_languages,
has_lied_resolution, has_lied_os, has_lied_browser, touch_support, js_fonts]
fingerParameters = {"fingerParameters": str(fingerParametersList)}{"messageBody":fingerParameters}
messageBodyDict = fingerParameters.copy()
messageBodyDict.update(clientType)
messageBodyDict.update(fingerId)
messageBodyDict.update(ip)
messageBodyDict.update(proxyFlag)
messageBodyDict.update(sessionId)
messageBodyDict.update(systemId)
messageBodyDict.update(timestamp)
messageBodyDict.update(appId)
messageBody = {"messageBody": str(messageBodyDict)}
# print(messageBody)
NewmessageBody = messageBody.copy()
NewmessageBody.update(topic)
NewmessageBody.update(tag)
NewmessageBody.update(key)
return NewmessageBodydef toJson(message):
return json.dumps(message, ensure_ascii=False)
def send(url, jsonbody, header):
print('发送的消息体为:%s' % jsonbody)
re = requests.post(url=url, data=jsonbody, headers=header).json()
return re
def getHeader():
'''获取请求头'''
header = {"Content-Type": "application/json"}
return header
class rcpTaskSet(TaskSet):
@task(1)
def fingerprint(self):
'''生产设备指纹'''
try:
sessionId_data = self.locust.sessionId_queue.get()
print(sessionId_data)
except queue.Empty:
print('sessionId data run out, test ended.')
exit(0)
try:
fingerId_data = self.locust.fingerId_queue.get()
print(fingerId_data)
except queue.Empty:
print('fingerId data run out, test ended.')
exit(0)
#self.messageDict(fingerId_data,sessionId_data)
json1 = toJson(messageDict(fingerId_data,sessionId_data))
send(RockMQ, json1, getHeader())
#保证并发测试数据唯一性，循环取数据
self.locust.sessionId_queue.put_nowait(sessionId_data)
self.locust.fingerId_queue.put_nowait(fingerId_data)
# payload = FG.messageDict( sessionId)
# json1 = FG.toJson(payload)
# self.send(self.RockMQ, json1, self.getHeader())
# @task
# def sendMQ(self):
# '''发送消息'''
# passclass rcpLocust(HttpLocust):
task_set = rcpTaskSet
sessionId_queue = queue.Queue()
for a in range(1000000):
sessionId = AutoGeneratedString(32)
sessionId_queue.put_nowait(sessionId)
fingerId_queue = queue.Queue()
for b in range(1000000):
fingerId = AutoGeneratedString(32)
fingerId_queue.put_nowait(fingerId)
min_wait = 1000
max_wait = 3000找到locust脚本，启动方式如下：
locust -H http://192.168.46.154:8765/topic/sendTopicMessage.do -f rcpLocust.py
最后在web上设置用户数和每秒加载的用户，done