septnancye

22.项目开发之量化交易抓取数据QuantTradeData(一)

项目创建及后端业务：定时更新“股票列表基础信息”数据

项目创建

该量化交易数据平台用于数据库的数据抓取、分析等操作。
和QuantTrade使用同一个数据库，无需重新创建

pom.xml


<properties>
   <java.version>1.8</java.version>
   <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
   <spring-boot.version>2.3.12.RELEASE</spring-boot.version>
</properties>
<dependencies>
   <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-web</artifactId>
   </dependency>
   <dependency>
       <groupId>org.mybatis.spring.boot</groupId>
       <artifactId>mybatis-spring-boot-starter</artifactId>
       <version>2.2.2</version>
   </dependency>
   <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>
       <version>8.0.33</version>
   </dependency>
   <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
       <optional>true</optional>
   </dependency>
   <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-test</artifactId>
       <scope>test</scope>
       <exclusions>
           <exclusion>
               <groupId>org.junit.vintage</groupId>
               <artifactId>junit-vintage-engine</artifactId>
           </exclusion>
       </exclusions>
   </dependency>
   <dependency>
       <groupId>io.springfox</groupId>
       <artifactId>springfox-boot-starter</artifactId>
       <version>3.0.0</version>
   </dependency>
   <dependency>
       <groupId>com.alibaba</groupId>
       <artifactId>fastjson</artifactId>
       <version>1.2.17</version>
   </dependency>
   <dependency>
       <groupId>com.opencsv</groupId>
       <artifactId>opencsv</artifactId>
       <version>5.7.1</version>
   </dependency>
   <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-quartz</artifactId>
   </dependency>
</dependencies>
<dependencyManagement>
   <dependencies>
       <dependency>
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-dependencies</artifactId>
           <version>${spring-boot.version}</version>
           <type>pom</type>
           <scope>import</scope>
       </dependency>
   </dependencies>
</dependencyManagement>

<build>
   <plugins>
       <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
           <version>3.8.1</version>
           <configuration>
               <source>1.8</source>
               <target>1.8</target>
               <encoding>UTF-8</encoding>
           </configuration>
       </plugin>
       <plugin>
           <groupId>org.springframework.boot</groupId>
           <artifactId>spring-boot-maven-plugin</artifactId>
           <version>${spring-boot.version}</version>
           <configuration>
               <mainClass>com.quanttrade_bk.QuantTradeBkApplication</mainClass>
               <skip>true</skip>
           </configuration>
           <executions>
               <execution>
                   <id>repackage</id>
                   <goals>
                       <goal>repackage</goal>
                   </goals>
               </execution>
           </executions>
       </plugin>
   </plugins>
</build>

application.properties


# 应用服务 WEB 访问端口
server.port=8082
# 配置数据库连接信息
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3380/db_quant?serverTimezone=UTC
spring.datasource.username=root
spring.datasource.password=1234
#配置mapper.xml的路径
mybatis.mapper-locations=classpath:mapper/*/*.xml

Application

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.web.client.RestTemplate;
import springfox.documentation.swagger2.annotations.EnableSwagger2;

@SpringBootApplication
@EnableSwagger2
public class QuantTradeDataApplication {

   public static void main(String[] args) {
       SpringApplication.run(QuantTradeDataApplication.class, args);
   }
   /**
    * 向spring注册RestTemplate工具
    * @return
    */
   @Bean
   public RestTemplate getRestTemplate(){
       return new RestTemplate();
   }
}

RestObject

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class RestObject {
   private Integer code;           //前端的状态码 0：业务失败  1：业务成功
   private String msg;              //前端需要展示的信息
   private Object data;            //前端需要的复杂数据
   //业务成功的方法----------
   /**
    * 业务成功，自定义返回msg和返回的数据
    * @param msg
    * @param data
    * @return
    */
   public static RestObject OK(String msg,Object data){
       return new RestObject(1,msg,data);
   }
   /**
    * 业务成功，自定义返回msg，无返回数据
    * @param msg
    * @return
    */
   public static RestObject OK(String msg){
       return OK(msg,null);
   }
   /**
    * 业务成功，自定义返回的数据，无返回msg
    * @param data
    * @return
    */
   public static RestObject OK(Object data){
       return OK(null,data);
   }
   //业务失败的方法----------
   /**
    * 业务失败，自定义返回msg和返回的数据
    * @param msg
    * @param data
    * @return
    */
   public static RestObject ERROR(String msg,Object data){
       return new RestObject(0,msg,data);
   }
   /**
    * 业务失败，自定义返回msg，无返回数据
    * @param msg
    * @return
    */
   public static RestObject ERROR(String msg){
       return ERROR(msg,null);
   }
   /**
    * 业务失败，自定义返回的数据，无返回msg
    * @param data
    * @return
    */
   public static RestObject ERROR(Object data){
       return ERROR(null,data);
   }
}

搭建Const和TuShareAPI工具

Const

public interface Const {
   //平台地址
   String TUSHARE_BASE_URL     =   "http://api.tushare.pro";
   //令牌
   String TUSHARE_TOKEN        =   "自己的token";
   //股票基本信息
   String STOCK_BASIC = "stock_basic";
   //股票日线基本行情
   String STOCK_DAILY = "daily";
}


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Component
public class TuShareAPI {
   @Autowired
   RestTemplate template;
   /**
    * 用于向tushare平台发送请求，并接收到响应
    * @param api
    * @param params
    * @param fields
    * @return
    */
   public JSONObject get(String api, Map<String, String> params, List<String> fields) {
       Map<String, Object> allParam = new HashMap<>();
       allParam.put("api_name", api);
       allParam.put("token", Const.TUSHARE_TOKEN);
       // 请求参数
       allParam.put("params", params);
       // 设置 需要响应回来的字段
       if (fields != null && fields.size()>0) {
           StringBuffer buffer = new StringBuffer();
           int size = fields.size();
           for (int i = 0; i <size; i++) {
               buffer.append(fields.get(i));
               if(i!=size-1){
                   buffer.append(",");
               }
           }
           allParam.put("fields", buffer.toString());
       }
       //设置并发送请求，获取并返回响应
       HttpHeaders headers = new HttpHeaders();
       headers.setContentType(MediaType.APPLICATION_JSON);
       HttpEntity<Map> entity = new HttpEntity<>(allParam, headers);
       String str = template.postForObject(Const.TUSHARE_BASE_URL,entity, String.class);
       JSONObject json = JSON.parseObject(str);
       return json;
   }
}

启动时，运行： http://localhost:8082/swagger-ui/index.html

后端业务：定时更新“股票列表基础信息”数据

需求说明

为了获取前一天的最新数据，我们需要每个工作日的晚上10点定时刷新stock_basic股票列表基础信息，并将最新数据插入或更新到数据库中，将已经退市的数据删除。

前置知识：Quartz定时框架

Apache提供定时框架Quartz来帮助我们进行定时任务。

依赖

<dependency>
   <groupId>org.springframework.boot</groupId>
   <artifactId>spring-boot-starter-quartz</artifactId>
</dependency>

其中作业：Job （其中去编写要执行的任务）
作业详情：JobDetail （对任务属性进行调整）
触发器：Trigger（负责执行具体的某个作业）都是执行任务必不可少的组件。

我们将需要执行的任务写入Job，并通过JobBuilder来构建JobDetail。
通过TriggerBuilder来构建触发器，并加入任务与执行周期&频率，让任务执行。
其中，Cron表达式无需独立学习，通过在线构建即可：https://cron.qqe2.com/

Job：

import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.scheduling.quartz.QuartzJobBean;

import java.util.Date;

public class TestJob1 extends QuartzJobBean {
   @Override
   protected void executeInternal(JobExecutionContext jobExecutionContext) throws JobExecutionException {
       //定时去做什么事情
       System.out.println("TestJob1:"+new Date().toLocaleString());
   }
}

任务配置：


import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class QuartzConfig {

   /**
    * 构建任务详情
    * @return
    */
   @Bean
   public JobDetail getTestJob1(){
       //1、通过构建，获取任务详情并返回
       //storeDurably()  脱离Trigger进行独立保存
       return JobBuilder.newJob(TestJob1.class).storeDurably().build();
   }

   @Bean
   public Trigger getTestJob1Trigger(JobDetail getTestJob1){
       //1、创建Cron表达式
       //1秒执行一次
//        CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("* * * * * ?");
       //从00秒开始  每10秒执行一次
       CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("0/10 * * * * ? ");
       //2、通过构建，完成触发器的创建并返回
       return TriggerBuilder.newTrigger().forJob(getTestJob1).withSchedule(cron).build();
   }
}

案例：定时点名每秒定时抽一个学生姓名

1、创建一个Job，读取txt里的数据，随机获取其中一个

import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.scheduling.quartz.QuartzJobBean;
import org.springframework.util.ResourceUtils;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

public class RandomNameJob extends QuartzJobBean {
   private static List<String> names = new ArrayList<>();
   static {
       //读取txt获取数据
       //获取file文件
       File file;
       try {
           file = ResourceUtils.getFile("classpath:test.txt");
       } catch (FileNotFoundException e) {
           throw new RuntimeException(e);
       }
       //获取缓冲字符流
       try (BufferedReader reader = new BufferedReader(new FileReader(file));){
           //读取数据，保存到names
           String line = null;
           while ((line=reader.readLine())!=null){
               names.add(line);
           }
       } catch (FileNotFoundException e) {
           throw new RuntimeException(e);
       } catch (IOException e) {
           throw new RuntimeException(e);
       }
   }
   private static Random r = new Random();
   @Override
   protected void executeInternal(JobExecutionContext jobExecutionContext) throws JobExecutionException {
       System.out.println("随机点名："+names.get(r.nextInt(names.size())));
   }
}

2、创建JobDetail，创建Trigger 给于cron

import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class QuartzConfig {

   /**
    * 构建任务详情
    * @return
    */
   @Bean
   public JobDetail getTestJob1(){
       //1、通过构建，获取任务详情并返回
       //storeDurably()  脱离Trigger进行独立保存
       return JobBuilder.newJob(TestJob1.class).storeDurably().build();
   }

   @Bean
   public Trigger getTestJob1Trigger(JobDetail getTestJob1){
       //1、创建Cron表达式
       //1秒执行一次
//        CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("* * * * * ?");
       //从00秒开始  每10秒执行一次
       CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("0/10 * * * * ? ");
       //2、通过构建，完成触发器的创建并返回
       return TriggerBuilder.newTrigger().forJob(getTestJob1).withSchedule(cron).build();
   }

   /**
    * 构建任务详情
    * @return
    */
   @Bean
   public JobDetail getRandomNameJob(){
       //1、通过构建，获取任务详情并返回
       //storeDurably()  脱离Trigger进行独立保存
       return JobBuilder.newJob(RandomNameJob.class).storeDurably().build();
   }

   @Bean
   public Trigger getRandomNameJobTrigger(JobDetail getRandomNameJob){
       //1、创建Cron表达式
       //1秒执行一次
       CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("* * * * * ?");
       //2、通过构建，完成触发器的创建并返回
       return TriggerBuilder.newTrigger().forJob(getRandomNameJob).withSchedule(cron).build();
   }
}

股票列表基础信息表（tb_stock_basic_info）

股票列表基础信息实体（StockBasicInfo）

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class StockBasicInfo {
   private String tscode;          //TS代码
   private String stockcode;    //股票代码
   private String stockname;   //股票名称
   private String stockarea;     //地域
   private String industry;       //所属行业
   private String cnspell;         //拼音缩写
   private String market;         //市场类型
   private String list_date;       //上市日期
   private String is_hs;            //是否沪深港通标的，N否 H沪股通 S深股通
   private String op_date;       //操作日期  yyyy-MM-dd
}

三层搭建

StockBasicInfoMapper


import org.springframework.stereotype.Repository;

@Repository
public interface StockBasicInfoMapper {
}

Application

import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.web.client.RestTemplate;
import springfox.documentation.swagger2.annotations.EnableSwagger2;

@SpringBootApplication
@EnableSwagger2
@MapperScan("com.quanttradedata.stock.mapper")
public class QuantTradeDataApplication {

   public static void main(String[] args) {
       SpringApplication.run(QuantTradeDataApplication.class, args);
   }
   /**
    * 向spring注册RestTemplate工具
    * @return
    */
   @Bean
   public RestTemplate getRestTemplate(){
       return new RestTemplate();
   }
}


StockBasicInfoMapper.xml
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
       PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
       "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.quanttradedata.stock.mapper.StockBasicInfoMapper">

</mapper>

StockService


import com.quanttradedata.stock.mapper.StockBasicInfoMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

@Service
public class StockService {
   @Autowired
   private StockBasicInfoMapper stockBasicInfoMapper;
}

StockController


import com.quanttradedata.stock.service.StockService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/stock")
@CrossOrigin //跨域在开发环境可以有，在生产环境中尽可能去除
public class StockController {
   @Autowired
   private StockService stockService;
}

业务实现

StockBasicInfoJob


import com.quanttradedata.stock.service.StockService;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.quartz.QuartzJobBean;
import org.springframework.stereotype.Component;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;

@Component
public class StockBasicInfoJob extends QuartzJobBean {
   @Autowired
   private StockService stockService;
   private SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd");

   @Override
   protected void executeInternal(JobExecutionContext jobExecutionContext) throws JobExecutionException {
       //1、准备需要传递给tuShare 参数
       HashMap<String, String> param = new HashMap<>();
       param.put("exchange","SSE");
       param.put("list_status","L");
       //2、请求 SSE上交所的股票基本信息
       System.out.println("请求SSE上交所的股票基本信息......");
       stockService.saveStockBasicInfoFromNet(param);
       //3、请求 SZSE深交所的股票基本信息
       System.out.println("请求 SZSE深交所的股票基本信息......");
       param.put("exchange","SZSE");
       stockService.saveStockBasicInfoFromNet(param);
       //4、数据更新完毕，删除退市失效数据
       stockService.delStockBasicInfosByOpDate(sdf1.format(new Date()));
   }
}

QuartzConfig


import com.quanttradedata.stock.job.StockBasicInfoJob;
import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class QuartzConfig {
   //股票基本信息的 任务详情创建 及 触发器创建
   @Bean
   public JobDetail getStockBasicInfoJob(){
       return JobBuilder.newJob(StockBasicInfoJob.class).storeDurably().build();
   }
   @Bean
   public Trigger getStockBasicInfoJobTrigger(JobDetail getStockBasicInfoJob){
       //1、编写cron表达式，指定触发的时间和周期
       //开发执行： 每个工作日的晚上22点
       //CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("0 0 22 ? * MON-FRI");
       //测试执行： 每5秒执行一次
       CronScheduleBuilder cron = CronScheduleBuilder.cronSchedule("0/5 * * * * ? ");
       //2、构建触发器，执行任务
       return TriggerBuilder.newTrigger().forJob(getStockBasicInfoJob).withSchedule(cron).build();
   }
}