量化：从okex获取k线数据，symbol（交易对）， 去重处理，存入到mongodb数据库

import json
import time
import pymongo
from threading import Thread
import pandas as pd
from urllib.request import Request, urlopen

pd.set_option('expand_frame_repr', False)
temp0 = []
temp1 = []
temp2 = []
temp3 = []


def get_url_data(url, retry_times=5):
    """
    从API接口中获取数据
    :param url: API接口pytho
    :param retry_times:最大尝试次数
    :return: API json格式数据
    """
    # while True:
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
        request = Request(url=url, headers=headers)
        b_data = urlopen(request, timeout=10).read()
        str_data = b_data.decode('utf-8')
        json_data = json.loads(str_data)
        return json_data
    except Exception as http_error:
        if retry_times > 0:
            return get_url_data(url, retry_times=retry_times - 1)
        else:
            print('尝试失败次数过多，已经放弃，错误信息：%s' % http_error)
            return None


def get_candle_from_okex(symbol, type='1min'):
    """
    从okex交易所获取k线数据
    :param symbol:  交易对
    :param type:  时间间隔
    :return:时间点、开盘价、最高价、最低价、收盘价和交易量
    """
    url = 'https://www.okex.com/api/v1/kline.do?symbol=%s&type=%s' % (symbol, type)  # k线数据
    # while True:
    json_data = get_url_data(url)
    if not json_data:
        print('获取数据错误！')
        return None
    info = json_data[-2]
    timestamp = info[0] / 1000
    time_array = time.localtime(timestamp)
    date = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
    open, high, low, close, volume = info[1:]
    msg = {'date': date, 'open': open, 'high': high, 'low': low, 'close': close, 'volume': volume}
    data = {'symbol': symbol, 'msg': msg}
    return data


def save_data_to_mongodb(symbol):
    """
    将数据存储在mongo数据库中
    :param symbol:  交易对
    :return:
    """
    client = pymongo.MongoClient(host='localhost', port=27017)
    db = client['dadada']
    coll = db[symbol]
    if symbol == 'bch_usdt':
        data = get_candle_from_okex(symbol)
        if data:
            date0 = data['msg']['date']
            print(date0, temp0)
            if temp0 != [] and date0 != temp0[0]:  # 去重处理
                temp0[0] = date0
                coll.insert(data)
                print('存入数据！')
            elif temp0 == []:
                print('temp0为空列表！')
                temp0.append(date0)
                coll.insert(data)
                print('temp0为空列表，第一次存入数据！')
            elif date0 == temp0[0]:
                print('抛弃数据！')
    if symbol == 'btg_usdt':
        data = get_candle_from_okex(symbol)
        if data:
            date1 = data['msg']['date']
            print(date1, temp1)
            if temp1 != [] and date1 != temp1[0]:
                temp1[0] = date1
                coll.insert(data)
                print('存入数据！')
            elif temp1 == []:
                temp1.append(date1)
                coll.insert(data)
                print('temp1为空列表, 第一次存入数据！')
            elif date1 == temp1[0]:
                print('抛弃数据！')
    if symbol == 'xrp_usdt':
        data = get_candle_from_okex(symbol)
        if data:
            date2 = data['msg']['date']
            print(date2, temp2)
            if temp2 != [] and date2 != temp2[0]:
                temp2[0] = date2
                coll.insert(data)
                print('存入数据！')
            elif temp2 == []:
                temp2.append(date2)
                coll.insert(data)
                print('temp2不为空，第一次存入数据')
            elif date2 == temp2[0]:
                print('抛弃数据！')
    if symbol == 'eos_usdt':
        data = get_candle_from_okex(symbol)
        if data:
            date3 = data['msg']['date']
            print(date3, temp3)
            if temp3 != [] and date3 != temp3[0]:
                temp3[0] = date3
                coll.insert(data)
                print('存入数据！')
            elif temp3 == []:
                temp3.append(date3)
                coll.insert(data)
                print('temp3不为空，第一次存入数据！')
            elif date3 == temp3[0]:
                print('抛弃数据！')
    else:
        pass


def main():
    while True:
        t1 = Thread(target=save_data_to_mongodb, args=('bch_usdt',))
        t1.start()
        t2 = Thread(target=save_data_to_mongodb, args=('btg_usdt',))
        t2.start()
        t3 = Thread(target=save_data_to_mongodb, args=('xrp_usdt',))
        t3.start()
        t4 = Thread(target=save_data_to_mongodb, args=('eos_usdt',))
        t4.start()
        t1.join()
        t2.join()
        t3.join()
        t4.join()


if __name__ == '__main__':
    main()

注意：代码冗余比较多。去重处理，设置全局变量用字典比用列表的效果要好许多。起线程可以用for in 循环创建指定数量的线程，代码要好看许多。