# -*- coding: utf-8 -*- from lxml import etree import pandas as pd import tushare as ts from redis import Redis import hashlib import pymysql # =============================================================调用stock_basic============================================================= def stockCode(): # 创建MySQL链接对象 conn1 = pymysql.connect(user='root', password='123456', database='stock', charset='utf8') cursor = conn1.cursor() # 创建redis链接对象 conn2 = Redis(host='127.0.0.1', port=6379) # 定义成功、失败条数 success = 0 fail = 0 exist = 0 # 定义空list code_list = [] # 获取数据 stock_basic = pro.stock_basic(list_status='L', fields='ts_code,name,area,exchange,list_date') for index, row in stock_basic.iterrows(): stk_code = row['ts_code'] # 股票代码 stk_name = row['name'] # 股票名称 city = row['area'] # 所属城市 exchange = row['exchange'] # 所属交易所 list_date = row['list_date'] # 上市日期 # xl_code = ts_code[-2:] + ts_code[:6] code_list.append(stk_code) return code_list # 将解析到的数据值生成一个唯一的标识进行redis存储 source = stk_code source_id = hashlib.sha256(source.encode()).hexdigest() # 将解析内容的唯一表示存储到redis的data_id中 ex = conn2.sadd('stk_code', source_id) # 获取没有爬取的内容 if ex == 1: try: sql1 = '''insert into stk_code_list(ts_code,symbol,xl_code) values('%s','%s','%s','%s','%s')''' % (stk_code,stk_name,city,exchange,list_date) cursor.execute(sql1) conn1.commit() success += 1 except: conn1.rollback() fail += 1 # 获取已经爬取的内容 else: exist += 1 # 关闭MySQL连接 cursor.close() conn1.close() print('程序抓取成功数据：%d条' % success) print('程序抓取失败数据：%d条' % fail) print('MySQL原来有数据：%d条' % exist) # =============================================================调用daily============================================================= def dailyData(): for i in range(0,12000,1000): daily = pro.daily(trade_date=current,fields='ts_code,open,high,low,close,pre_close,pct_chg,amount',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用daily_basic============================================================= def dailyBasic(): for i in range(0,12000,1000): daily = pro.daily(trade_date=current,fields='ts_code,trade_date,turnover_rate_f,volume_ratio,pe_ttm,pb,ps_ttm,dv_ttm,total_share,float_share,total_mv,circ_mv,',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用moneyflow============================================================= def moneyFlow(): for i in range(0,12000,1000): daily = pro.daily(trade_date=current,fields='ts_code,trade_date,buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用limit_list============================================================= def limitData(): for i in range(0,12000,1000): daily = pro.daily(trade_date=current,fields='ts_code,trade_date,fc_ratio,fd_amount,first_time,last_time,open_times,strth,limit',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用moneyflow_hsgt============================================================= def moneyflowHsgt(): for i in range(0,12000,1000): daily = pro.daily(trade_date=current,fields='trade_date,hgt,sgt,north_money',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用top10_floatholders============================================================= def topFloatholders(): for i in range(0,12000,1000): daily = pro.daily(period='20191231',fields='ts_code,end_date,holder_name,hold_amount',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用top_inst============================================================= def topIinst(): for i in range(0,12000,1000): daily = pro.daily(trade_date,fields='ts_code,trade_date,exalter,buy,sell,net_buy',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用share_float============================================================= def shareFloat(): for i in range(0,12000,1000): daily = pro.daily(float_date,fields='ts_code,float_date,float_share',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用stk_holdernumber============================================================= def stkHoldernumber(): for i in range(0,12000,1000): daily = pro.daily(float_date,fields='ts_code,float_date,float_share',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用stk_holdertrade============================================================= def stkHoldertrade(): for i in range(0,12000,1000): daily = pro.daily(float_date,fields='ts_code,float_date,float_share',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # =============================================================调用index_daily============================================================= def indexDaily(): for i in range(0,12000,1000): daily = pro.daily(float_date,fields='ts_code,float_date,float_share',offset=i,limit=1000) for index, row in daily.iterrows(): stk_code = row['ts_code'] # 股票代码 open = row['open'] # 开盘价 high = row['high'] # 最高价 low = row['low'] # 最低价 close = row['close'] # 收盘价 pre_close = row['pre_close'] # 昨收价 pct_chg = row['pct_chg'] # 涨跌幅 amount = row['amount'] # 成交额 # ================================================================主函数================================================================ if __name__ == '__main__': # 初始化tushare.pro接口 pro = ts.pro_api('ac16b470869c5d82db5033ae9288f77b282d2b5519507d6d2c72fdd7') code_list = stockCode() # 创建 retry_count：当网络异常后重试次数，默认为3 pause: 重试时停顿秒数，默认为0
