MyBatis-Plus+EasyPOI大量数据导出

MyBatis-Plus的入门

Pom依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <!-- pom模型版本 -->
    <modelVersion>4.0.0</modelVersion>

    <!-- 项目信息 -->
    <groupId>com.mad</groupId>              <!-- 项目唯一标识 -->
    <artifactId>mybatis-plus</artifactId>   <!-- 项目名 -->
    <version>1.0-SNAPSHOT</version>         <!-- 版本 -->
    <packaging>jar</packaging>              <!-- 打包方式 （pom,war,jar） -->

    <!-- 定义父目录继承关系 -->
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.0.2.RELEASE</version>
    </parent>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <!--mybatis-plus-->
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.3.0</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
            <version>2.0.2.RELEASE</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>1.1.9</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.46</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.8</version>
            <scope>provided</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <!-- maven插件 -->
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <version>2.0.2.RELEASE</version>
            </plugin>
        </plugins>
    </build>
</project>

项目结构

配置文件

application.properties

#定义服务器端口
server.port=8080
#定义tomcat语言编码
server.tomcat.uri-encoding=utf-8

spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3306/mydb
spring.datasource.username=root
spring.datasource.password=root

SpringBoot启动类

在 Spring Boot 启动类中添加 @MapperScan 注解，扫描 Mapper 文件夹

package com.mad;


import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
@MapperScan(value = "com.mad.mapper")
public class MyBatisPlusApplication {

    public static void main(String[] args) {
        SpringApplication.run(MyBatisPlusApplication.class,args);
    }
}

实体类Student

// 由于此处类名与表名一致，未使用注解指定表名
// @TableName(value = "tb_name")
@Data
public class Student {
    // value与数据库主键列名一致，若实体类属性名与表主键列名一致（或者符合驼峰命名）可省略value
    @TableId(value = "sid",type = IdType.AUTO)  //指定自增策略
    private Integer sid;

    private String sname;

    private Integer age;

    private String sex;

    private String department;

    private String address;

    private String birthplace;
}

Mapper文件

package com.mad.mapper;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.mad.entity.Student;

public interface StudentMapper extends BaseMapper<Student> {

}

测试类（单表操作）

实际测试时直接使用注解 @SpringBootTest ，studentMapper会注入失败，需要指定classes = MyBatisPlusApplication.class
其中 students.forEach(System.out::println); 是JDK8新增的语法，类似 lambda的语法糖
此版本的条件查询使用的是QueryWrapper，之前的版本是EntityMapper

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.OrderItem;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.mad.MyBatisPlusApplication;
import com.mad.entity.Student;
import com.mad.mapper.StudentMapper;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


@RunWith(SpringRunner.class)
@SpringBootTest(classes = MyBatisPlusApplication.class)
public class SampleTest {
    @Autowired
    StudentMapper studentMapper;

    // mybatis-plus的单表操作
    @Test
    public void test(){
        // 根据id查询时需要给实体类的对应主键的属性添加注解 @TableId
        // 1、根据id查询单条
        Student student = studentMapper.selectById(1);
        System.out.println(student);
        
        // 2、根据id批量查询
        List<Integer> idList = new ArrayList<>();
        idList.add(2);
        idList.add(3);
        List<Student> ids = studentMapper.selectBatchIds(idList);
        ids.forEach(i -> System.out.println("根据id批量查询:"+i));
        
        // 3、无条件查询所有数据
        List<Student> students = studentMapper.selectList(null);
//        students.forEach(System.out::println);
        students.forEach(i -> System.out.println("无条件查询所有数据:"+i));
        
        // 4、使用Map封装条件参数查询
        Map<String,Object> map = new HashMap<>();
        map.put("sname","小王");
        map.put("age",18);
        List<Student> mapList = studentMapper.selectByMap(map);
        mapList.forEach(i -> System.out.println("使用Map封装条件参数查询:"+i));
        
        // 5、无条件分页查询
        Page<Student> studentPage = studentMapper.selectPage(new Page<>(1, 3), null);
        List<Student> records = studentPage.getRecords();
        List<OrderItem> orders = studentPage.getOrders();
        records.forEach(i -> System.out.println("无条件分页查询:"+i));
        orders.forEach(i -> System.out.println("无条件分页查询orders:"+i));
        
        // 6、条件查询单条数据(若查询结果为多条则报错：TooManyResultsException: Expected one result (or null) to be returned by selectOne(), but found: 2)
        Student wrapper01 = studentMapper.selectOne(new QueryWrapper<Student>()
                .eq("sname", "小王")
                .like("department", "信息")
                .eq("sid",1));
        System.out.println("条件查询单条数据:"+wrapper01);
        
        // 7、条件查询多条数据 且 指定结果集
        List<Student> wrapper02 = studentMapper.selectList(new QueryWrapper<Student>()
                .select("sname", "address")
                .between("age", 20, 25)
                .ne("sex", "女"));
        wrapper02.forEach(i -> System.out.println("条件查询多条数据-指定结果集:"+i));
        
        // 8、根据id更新
        Student update01 = new Student();
        update01.setSid(1);
        update01.setDepartment("信息工程");
        studentMapper.updateById(update01);
        
        // 9、根据条件更新
        Student update02 = new Student();
        update02.setAddress("四川");
        studentMapper.update(update02 , new UpdateWrapper<Student>()
                .eq("address","重庆"));
        // delete 和 insert 同理，可根据方法名直接理解如何使用
    }
}

使用EasyPOI导出大量数据

Pom依赖

阿里的EasyExcel尚未了解

    <dependencies>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.8</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-jdbc</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <scope>runtime</scope>
        </dependency>

        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>3.3.1.tmp</version>
        </dependency>

        <!-- 阿里开源EXCEL -->
<!--        <dependency>-->
<!--            <groupId>com.alibaba</groupId>-->
<!--            <artifactId>easyexcel</artifactId>-->
<!--            <version>1.1.2-beta4</version>-->
<!--        </dependency>-->

        <dependency>
            <groupId>cn.afterturn</groupId>
            <artifactId>easypoi-base</artifactId>
            <version>3.3.0</version>
        </dependency>
        <dependency>
            <groupId>cn.afterturn</groupId>
            <artifactId>easypoi-web</artifactId>
            <version>3.3.0</version> </dependency>
        <dependency>
            <groupId>cn.afterturn</groupId>
            <artifactId>easypoi-annotation</artifactId>
            <version>3.3.0</version>
        </dependency>
    </dependencies>

启动类

配置文件没什么说的 连接数据库完事

@SpringBootApplication
@MapperScan("com.mad.mapper")
public class MySpringBootApplication {
    public static void main(String[] args) {
        SpringApplication.run(MySpringBootApplication.class,args);
    }

    // 配置Mybatis-Plus的分页拦截器
    @Bean
    public PaginationInterceptor paginationInterceptor() {
        PaginationInterceptor page = new PaginationInterceptor();
        // 设置最大单页限制数量，默认 500 条，小于 0 如 -1 不受限制;
        page.setLimit(0);
        return page;
    }
}

导出类

@Data
public class Student {
    @Excel(name = "主键", orderNum = "0", width = 15)
    private Integer id;
    @Excel(name = "姓名", orderNum = "1", width = 15)
    private String name;
    @Excel(name = "年龄", orderNum = "2", width = 15)
    private Integer age;
    // 导入时男女和01交换位置
    @Excel(name = "性别", orderNum = "3", width = 15, replace = { "男_0", "女_1" })
    private Integer sex;
    @Excel(name = "生日", orderNum = "4", width = 15, databaseFormat = "yyyyMMddHHmmss", format = "yyyy-MM-dd HH:mm:ss")
    private Date birth;
    @Excel(name = "创建时间", orderNum = "5", width = 15,databaseFormat = "yyyyMMdd", format = "yyyy-MM-dd")
    private Date createTime;
    @Excel(name = "成绩", orderNum = "6", width = 15)
    private Double score;
}

Mapper

@Repository
public interface StudentMapper extends BaseMapper<Student> {}

控制层

为方便测试没有写Service层
其实就是利用EasyPoi的方法： public static Workbook exportBigExcel(ExportParams entity, Class pojoClass, Collection dataSet)，可以发现其实使用的是 this.workbook = new SXSSFWorkbook();
本例为一次查询20000条记录，将数据依次append到workbook中，实测20w数据导出耗时几秒

@RestController
public class MyController {

    @Resource
    StudentMapper studentMapper;

    @GetMapping(value = "/export")
    public void exportAllSalesRecordSellIn(HttpServletResponse response) throws Exception {
        Long t1 = System.currentTimeMillis();
        System.out.println();
        ExportParams params = new ExportParams("测试数据统计", "测试数据");
        Workbook workbook = null;
        Integer integer = studentMapper.selectCount(null);
        System.out.println("总条数："+integer);
        int totalPage = (int)Math.ceil((double)integer/20000);
        for(int currentPage = 1; currentPage <= totalPage; currentPage++){
            Long l1 = System.currentTimeMillis();
            // mybatis的分页
            Page<Student> page = new Page<>(currentPage,20000);
            Page<Student> studentPage = studentMapper.selectPage(page, null);
            Long l2 = System.currentTimeMillis();
            System.out.println("查询"+currentPage+"耗时:"+(l2-l1)/1000+"秒");
            // 会进行数据的append操作 将所有数据分批写入workbook 防止内存溢出
            workbook = ExcelExportUtil.exportBigExcel(params, Student.class, studentPage.getRecords());
        }
        ExcelExportUtil.closeExportBigExcel();

        String fileName = "test.xlsx";
        //告诉浏览器下载excel
        downloadExcel(fileName, workbook, response);
        Long t2 = System.currentTimeMillis();
        System.out.println("总共耗时:"+(t2 - t1)/1000+"秒");
    }

    protected void downloadExcel(String filename, Workbook workbook, HttpServletResponse response) throws Exception {
        response.setContentType("application/vnd.ms-excel");
        response.setHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(filename, "utf-8"));
        OutputStream outputStream = response.getOutputStream();
        workbook.write(outputStream);
        outputStream.flush();
        outputStream.close();
    }

    // 分Sheet导出待测试
}

数据库中插入百万测试数据

表结构

CREATE TABLE `student` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `name` varchar(50) DEFAULT NULL,
  `age` int(2) DEFAULT NULL,
  `sex` int(1) DEFAULT NULL,
  `birth` date DEFAULT NULL,
  `create_time` datetime DEFAULT NULL,
  `score` decimal(8,2) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8

生成随机字符串函数

DROP FUNCTION IF EXISTS `rand_string`;
CREATE  FUNCTION `rand_string`(n INT) RETURNS varchar(255) CHARSET utf8
BEGIN 
    DECLARE char_str varchar(200) DEFAULT '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
    DECLARE return_str varchar(255) DEFAULT '';
    DECLARE i INT DEFAULT 0;
    WHILE i < n DO
        SET return_str = concat(return_str, substring(char_str, FLOOR(1 + RAND()*36), 1));
        SET i = i+1;
    END WHILE;
    RETURN return_str;
END

存储过程批量插入

DROP PROCEDURE IF EXISTS `batchInsertTestData`;
CREATE PROCEDURE `batchInsertTestData` (n INT) 
BEGIN 
    DECLARE i INT DEFAULT 0;
    WHILE i < n DO
				insert into student(name,age,sex,birth,create_time,score) 
				values(rand_string(4),ROUND(RAND()) , ROUND(RAND()*100),curdate(),now(),FLOOR(RAND()*100));
        SET i = i+1;
    END WHILE;
END

调用存储过程生成数据

call batchInsertTestData(1000000);	//插入1000000条数据

优化

实测在Mysql存储引擎为innoDB的情况下，使用存储过程批量插入很慢，因此需要进行优化：
1. 在插入数据前检查表是否存在索引，若存在先删除，插入数据完成后再重新设置索引
2. 设置存储引擎为MyISAM，插入数据完成后根据需要设置引擎
3. 在本次测试使用的存储过程中，是在循环中多次进行insert into values 操作，可修改为insert into values(),values()… 把values拼接再插入，但注意一次insert对values的次数有限制 此处未测试