bitbank 获取实时深度的算法

一,深度就是这个,上边是卖盘(asks),下边是买盘(bids)

二,官方文档,深度的订阅一共有两个,增量数据(Depth Diff),和全量数据(Depth Whole),都需要订阅,具体交互格式可以看文档

文档上没写的东西

1,ping只要发送字符串2就可以

2,全量数据每15秒推送一次

3,全量推送的深度是200档

4,bitbank网站也是用的这个接口,可以打开浏览器开发者模式,查看交互细节

三,大概思路

    1).接收交易所返回的深度数据,区分全量,增量,整理成统一的格式

    2).接收第一步整理好的格式,创建或维护深度列表(每次接收全量推送,都会重新创建深度列表,增量数据则更新深度列表)

四,代码,估计其他交易所的深度同步,也差不多

# coding=utf-8

import time, decimal, sys, redis, math, pymysql,importlib, requests, hmac, base64, datetime, json, threading, zlib, pickle,traceback
import dateutil.parser as dp
from decimal import *
from conf.nodeconf import NodeConf
from pub.wsserver.wsserver import wsserver
from pub.base.connect import redis_pool, mysql_pool
from pub.base.dmsg import dmsg
from makermarket.wsserver.marketwsserver import marketwsserver
from clients.util import get_sub_str
from clients.util import get_future_name
import time,datetime
from datetime import datetime, timedelta
from decimal import Decimal as D
import hashlib


class bitbankwsserver(marketwsserver):
    def __init__(self, tcode, service_name, runpack, wsurl):
        super(bitbankwsserver, self).__init__(tcode, service_name, runpack, wsurl)

        tokenpairstr = self.nodeconf.get(self.tcode, "tokenpairs")
        tarr = tokenpairstr.split(",")
        self.tokens = list()
        self.run_time = int(time.time() * 1000)
        self.is_heartbeat = False
        for i in tarr:
            ar = i.split("_")
            m = dict()
            m["token"] = ar[0]
            m["quote"] = ar[1]
            self.tokens.append(m)

        self.next = False
        self.cache = {}

    def on_open(self, ws):
        pass
        # self.subscribe(ws)
        # timer = threading.Timer(1.0, self.heartbeat, [ws])
        # timer.start()

    def heartbeat(self, ws):
        ws.send("2")
        timer = threading.Timer(10, self.heartbeat, [ws])
        timer.start()

    def inflate(self, data):
        decompress = zlib.decompressobj(-zlib.MAX_WBITS)
        inflated = decompress.decompress(data)
        inflated += decompress.flush()
        return inflated

    def on_message(self, ws, msg):
        try:
            # self._l.info('msg={}'.format(str(msg)))
            if(msg == '40' or msg == '3'):
               return

            if(msg.startswith('0')):
                self.subscribe(ws)
                timer = threading.Timer(0, self.heartbeat, [ws])
                timer.start()

            if(msg.startswith('42')):
                msg = json.loads(msg[2:])
                # self._l.info('msg='+str(msg))

                ar = msg[1]["room_name"].split("_")
                token = dict()
                token["token"] = ar[2].lower()
                if(token["token"] == "bcc"):
                    token["token"] = "bch"
                token["quote"] = ar[3].lower()
                token['contract_type'] = ''

                l = self.parse_depth(msg[1]['message'],msg)

                depth_list = self.depth_start(l)

                depth_format = self.depth_format(depth_list)
                
                self.save_depth(depth_format,token)

        except Exception as e:
            self._l.error('错误信息:'+str(e)+'|出错文件:'+e.__traceback__.tb_frame.f_globals['__file__']+'|出错行号:'+str(e.__traceback__.tb_lineno))
    
    def parse_depth(self, result,recv):
        event = "all"
        item = {'asks': [], 'bids': []}
        item['time'] = result['data'].get('timestamp', int(time.time() * 1000))
        for k, v in result['data'].items():
            if k not in ['asks', 'bids', 'a', 'b']:
                continue
            for rec in v:
                item['asks' if k in ['asks', 'a'] else 'bids'].append({'price': float(rec[0]), 'size': float(rec[1])})
        if "depth_diff_" in recv[1]['room_name']:
            event = "add"
            item['time'] = result['data'].get('t', int(time.time() * 1000))

        room_name_arr = recv[1]['room_name'].split('_')
        return {"db_name":'bitbank',
                "table_name":room_name_arr[2]+'_'+room_name_arr[3],
                "data":item,
                "event":event,
                "list_length":200,
                "match":True }

    def subscribe(self, ws):
        #需要订阅多个币种,多发送几次订阅字符串就可以
        for pair in self.tokens:
            sub = pair['token']+'_'+pair['quote']
            #全量深度,15秒推送一次,200档
            sub_all  = ["join-room","depth_whole_"+sub]
            sub_string = '42'+json.dumps(sub_all)
            ws.send(sub_string)
            self._l.info('send={}'.format(sub_string))
            #增量深度,有更新时推送
            sub_diff = ["join-room","depth_diff_"+sub]
            sub_string = '42'+json.dumps(sub_diff)
            ws.send(sub_string)
            self._l.info('send={}'.format(sub_string))

    def depth_format(self,m):
        try:
            bidlist = list()
            for bid in m["bids"]:
                t = dict()
                t["price"] = float(bid['price'])
                t["size"] = float(bid['size'])
                t["sizeu"] = float(bid['price']) * float(bid['size'])
                t["original_size"] = float(bid['size'])
                bidlist.append(t)

            asklist = list()
            for ask in m["asks"]:
                t = dict()
                t["price"] = float(ask['price'])
                t["size"] = float(ask['size'])
                t["sizeu"] = float(ask['price']) * float(ask['size'])
                t["original_size"] = float(ask['size'])
                asklist.append(t)

            pack = dict()
            pack["bids"] = bidlist
            pack["asks"] = asklist
            pack["ts"] = int(m['time']/1000)
            pack["ts_ms"] = m['time']

            return pack
        except Exception as e:
            self._l.error('错误信息:'+str(e)+'|出错文件:'+e.__traceback__.tb_frame.f_globals['__file__']+'|出错行号:'+str(e.__traceback__.tb_lineno))

    def depth_start(self,parse_data):
        try:
            table_name = parse_data.get("table_name","").lower()#table_name
            db_name = parse_data.get("db_name")#db_name
            item = parse_data.get("data")#数据
            list_length = parse_data.get("list_length",1000)#数据
            event = parse_data.get("event","add")#数据是增量还是全量 all 全量 add 增量
            update_key = parse_data.get("update_key","price")#数据是增量还是全量 all 全量 add 增量
            match = parse_data.get("match",False)#是否进行自撮合
            recv_delay = int(time.time()*1000) - int(item["time"])#接收的数据和当前时间的延迟
            start_time = time.time()*1000
            self._l.info("recvie %s ,%s %s %s " %  (db_name,table_name,str(len(item.get("bids",[]))),str(len(item.get("asks",[])))))
            redis_item = json.loads(json.dumps(item))

            #增量数据
            if event == 'add':
                #维护深度列表,更新深度价格的数量,如果是0,则中深度列表中删除.如果还没有深度列表,则创建深度列表,但是此深度中的买一和卖一价格,是不准确的
                redis_item = self.update_data(table_name,redis_item,update_id = update_key )
            redis_item["crawl_time"] = time.time()*1000
            redis_item["market"] = db_name
            redis_item["symbol"] = table_name

            #全量数据
            if redis_item is not None :
                if match :
                    '''
                        当为全量数据时,则创建深度列表,当为增量数据时,则维护深度列表
                        创建或维护深度列表,从买盘和卖盘中删除价格相同,数量相同的项,这种操作是针对增量数据做的
                        两次全量数据之前,有15秒的时间,都是增量在推数据,增量推的数据,在更新后,在深度列表中,会有买盘价格比卖盘价格高的情况,所有需要这个操作
                    '''
                    redis_item = self.match_depth(table_name,redis_item)
                   #redis_item = {'bids':[{'price':10,'size':0.4845}...],'asks':[{'price':10,'size':0.4845}...]}
                #对买盘的深度列表降序排序,对卖盘的深度列表升序排序
                redis_item = self.sort_data( json.loads(json.dumps(redis_item)) ,list_length = list_length )
                
                process_cost = int(time.time()*1000-start_time) #数据处理耗时
                self.set_last_item( table_name,json.loads(json.dumps(redis_item)) )
                self._l.info('size0={},ask0={}'.format(redis_item['asks'][0]['size'],redis_item['asks'][0]['price']))
                self._l.info('bid0={},size0={}'.format(redis_item['bids'][0]['price'],redis_item['bids'][0]['size']))
                self._l.info("%s ,%s save_cost:%s " %  (db_name,table_name,str(int(time.time()*1000-start_time))))
                return redis_item
        except:
            traceback.print_exc()
            return False
        return True

    def update_data(self,symbol,result,update_id = "price" ):
        # result = {'bids':[{'price':10,'size':0.3843}],'asks':[{'price':10,'size':0.3843}]}
        '''增量数据更新
        '''
        last_depth = self.get_last_item(symbol)
        #last_depth = {'bids':[{'price':10,'size':0.4845}...],'asks':[{'price':10,'size':0.4845}...]}
        bids_price_size_dict = {} 
        #将上次bid数据按update_id放入 dict中
        if last_depth is not None and 'bids' in last_depth:
            for bid in last_depth['bids']:
                bids_price_size_dict[ bid[update_id] ] = bid
                # bids_price_size_dict = {10:{'price':10,'size':0.4845}}

        #遍历本次数据
        if 'bids' in result:
            for bid in result['bids']:
                if bid['size'] < 0:
                    bid['size'] = bids_price_size_dict[ bid[update_id] ]["size"] + bid['size']
                bids_price_size_dict[ bid[update_id] ] = bid
                if(float(bids_price_size_dict[ bid[update_id] ]['size'])  == 0 ):
                    #如果推送的深度size,为0,说明这一单被吃光了,从列表里删除
                    del bids_price_size_dict[ bid[update_id] ]

        #将上次ask数据按update_id放入 dict中
        asks_price_size_dict = {}
        if last_depth is not None and 'asks' in last_depth:
            for ask in last_depth['asks']:
                asks_price_size_dict[ask[update_id]] = ask

        #遍历本次数据
        if 'asks' in result:
            for ask in result['asks']:
                if ask['size'] < 0:
                    ask['size'] = asks_price_size_dict[ask[update_id]]['size'] + ask['size']
                asks_price_size_dict[ask[update_id]] = ask #更新上次数据
                if(float(asks_price_size_dict[ask[update_id]]['size'])  == 0 ):
                    del asks_price_size_dict[ask[update_id]]

        #以上四个步骤,为了更新深度价格的数量,如果是0,则从深度列表中删除

        result_bids = []
        for update_id, item in bids_price_size_dict.items():
            result_bids.append(item)

        result_asks = []
        for update_id, item in asks_price_size_dict.items():
            result_asks.append(item)
        return { 'bids' : result_bids, 'asks' : result_asks ,'time': result["time"] }
    
    def match_depth(self,table_name , depth_data):
        """撮合深度数据
        """
        asks = [{"price":x["price"],"size":D(str(x["size"])) } for x in depth_data["asks"]]
        bids = [{"price":x["price"],"size":D(str(x["size"])) } for x in depth_data["bids"]]
        ask_dict = { x["price"]: D( str(x["size"]) ) for x in asks }
        bid_dict = { x["price"]: D( str(x["size"]) ) for x in bids }
        #bdis = [{'price':10,'size':0.233}...]
        #bid_dict = {10:0.233...}
        #主动计算买卖盘头部的数据是否能抵消,如果能抵消,则从深度列表里删除
        for b in bids:#遍历买单数据，（高->低）
            need_size = b["size"] #买量
            price = b["price"] #买价
            #遍历买盘深度的每一个价格,如果这个价格能买到卖盘的币,则保留这个卖盘深度
            a_list = [ a for a in asks if a["price"] <= price ] #获取能买到的卖单
            if len(a_list) == 0:
               break
            a_list.sort(key=lambda x : x['price']) #低->高
            for a in a_list : #遍历卖单，(低->高)
                if need_size <= 0 : #如果买单量<=0 则停止
                   break
                self._l.info( table_name + " 可撮合买单价格"+str(b)+"可撮合卖单价格"+str(a) )
                if a["size"] < need_size:# 卖单的量<买单的量 删掉卖单数据，更新买单数据
                    need_size -= a["size"]
                    del ask_dict[a["price"]]
                    bid_dict[price] = need_size
                elif a['size'] == need_size: #买卖单量相等 删掉买单 和 卖单
                    del ask_dict[a["price"]]
                    del bid_dict[price]
                    need_size = D(0)
                else:#卖单的量>买单的量 删掉买单数据，更新卖单数据
                    del bid_dict[price]
                    ask_dict[a["price"]] = a["size"] - need_size
                    need_size = D(0)
            asks = [{"price":k,"size":v} for k,v in ask_dict.items()] #更新能买到的卖单
        asks = [{"price":k,"size":float(str(v)) } for k,v in ask_dict.items()] #更新能买到的卖单
        bids = [{"price":k,"size":float(str(v)) } for k,v in bid_dict.items()] #更新能买到的卖单
        depth_data["bids"] = bids
        depth_data["asks"] = asks
        depth_data["bids"].sort(key=lambda x : x['price'], reverse=True)
        depth_data["asks"].sort(key=lambda x : x['price'])
        depth_data["md5"] = 1#self.get_data_md5(depth_data)
        return depth_data

    def sort_data(self,item,list_length = 100):
        '''排序
        '''
        item['bids'].sort(key=lambda x : x['price'], reverse=True)
        item['asks'].sort(key=lambda x : x['price'])
        item['bids'] = item['bids']#[:list_length]
        item['asks'] = item['asks']#[:list_length]
        return item
        
    def set_last_item(self,symbol,item):
        '''更新深度数据缓存
        '''
        item["bids"] = item["bids"][:500]
        item["asks"] = item["asks"][:500]
        self.cache["last_item_"+symbol.lower()] = item
    
    def get_last_item(self,symbol):
        '''获取上一次的深度item
        '''
        return self.cache.get("last_item_"+symbol.lower())

    def get_data_md5(self,item):
        #data = {k: v for k, v in item.items() if k not in ['_id', 'time','md5']}
        data = {"bids":item["bids"],"asks":item["asks"]}
        if data == {}:
            return None
        return hashlib.md5(json.dumps(data)).hexdigest()