SpringBoot使用stream查询实现Mysql百万级数据导出
SpringBoot使用stream查询实现Mysql百万级数据导出
一 JPA实现百万级数据导出
参考博客：http://knes1.github.io/blog/2015/2015-10-19-streaming-mysql-results-using-java8-streams-and-spring-data.html
实体
public class Authors {
private Integer id;
private String firstName;
private String lastName;
private String email;
private Date birthdate;
private Date added;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName == null ? null : firstName.trim();
}
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName == null ? null : lastName.trim();
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email == null ? null : email.trim();
}
public Date getBirthdate() {
return birthdate;
}
public void setBirthdate(Date birthdate) {
this.birthdate = birthdate;
}
public Date getAdded() {
return added;
}
public void setAdded(Date added) {
this.added = added;
}
@Override
public String toString() {
return this.id + "," + this.firstName + "," + this.lastName + "," + this.email + "," + this.birthdate + "," + this.added;
}
}
Repository
方法的返回类型定义成Stream。Integer.MIN_VALUE告诉jdbc driver逐条返回数据
@QueryHints(value = @QueryHint(name = HINT_FETCH_SIZE, value = "" + Integer.MIN_VALUE))
@Query(value = "select t from Authors t")
Stream<Authors> streamAll();
Controller
@RequestMapping(value = "/todos.csv", method = RequestMethod.GET)
@Transactional(readOnly = true)
public void exportTodosCSV(HttpServletResponse response) {
response.addHeader("Content-Type", "application/csv");
response.addHeader("Content-Disposition", "attachment; filename=todos.csv");
response.setCharacterEncoding("UTF-8");
try(Stream<Authors> todoStream = todoRepository.streamAll()) {
PrintWriter out = response.getWriter();
todoStream.forEach(rethrowConsumer(todo -> {
String line = todo.toString();
out.write(line);
out.write("\n");
entityManager.detach(todo);
}));
out.flush();
} catch (IOException e) {
log.info("Exception occurred " + e.getMessage(), e);
throw new RuntimeException("Exception occurred while exporting results", e);
}
}
Util工具类
import java.util.function.Consumer;
import java.util.function.Function;
/**
* Utility class to avoid try/catch boiler plate by PaoloC (http://stackoverflow.com/users/2365724/paoloc).
*
* Taken from here:
* http://stackoverflow.com/questions/27644361/how-can-i-throw-checked-exceptions-from-inside-java-8-streams/30974991#30974991
*
* Created on: 10/10/15
*/
public final class LambdaExceptionUtil {
@FunctionalInterface
public interface Consumer_WithExceptions<T, E extends Exception> {
void accept(T t) throws E;
}
@FunctionalInterface
public interface Function_WithExceptions<T, R, E extends Exception> {
R apply(T t) throws E;
}
/**
* .forEach(rethrowConsumer(name -> System.out.println(Class.forName(name))));
*/
public static <T, E extends Exception> Consumer<T> rethrowConsumer(Consumer_WithExceptions<T, E> consumer) throws E {
return t -> {
try {
consumer.accept(t);
} catch (Exception exception) {
throwActualException(exception);
}
};
}
/**
* .map(rethrowFunction(name -> Class.forName(name))) or .map(rethrowFunction(Class::forName))
*/
public static <T, R, E extends Exception> Function<T, R> rethrowFunction(Function_WithExceptions<T, R, E> function) throws E {
return t -> {
try {
return function.apply(t);
} catch (Exception exception) {
throwActualException(exception);
return null;
}
};
}
@SuppressWarnings("unchecked")
private static <E extends Exception> void throwActualException(Exception exception) throws E {
throw (E) exception;
}
}
二 Mybatis实现百万级数据导出
mapper
MyBatis实现逐条获取数据，必须要自定义ResultHandler，然后在mapper.xml文件中，对应的select语句中添加fetchSize=“-2147483648”,最后将自定义的ResultHandler传给SqlSession来执行查询，并将返回的结果进行处理
<select id="selectByExample" parameterType="com.alphathur.mysqlstreamingexport.domain.AuthorsExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
'false' as QUERYID,
<include refid="Base_Column_List" />
from authors
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
<select id="streamByExample" fetchSize="-2147483648" parameterType="com.alphathur.mysqlstreamingexport.domain.AuthorsExample" resultMap="BaseResultMap">
select
<if test="distinct">
distinct
</if>
'false' as QUERYID,
<include refid="Base_Column_List" />
from authors
<if test="_parameter != null">
<include refid="Example_Where_Clause" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
</select>
public interface AuthorsMapper {
List<Authors> selectByExample(AuthorsExample example);
List<Authors> streamByExample(AuthorsExample example); //以stream形式从mysql获取数据
}
service
@Service
public class AuthorsService {
private final SqlSessionTemplate sqlSessionTemplate;
private final AuthorsMapper authorsMapper;
public AuthorsService(SqlSessionTemplate sqlSessionTemplate, AuthorsMapper authorsMapper) {
this.sqlSessionTemplate = sqlSessionTemplate;
this.authorsMapper = authorsMapper;
}
/**
* stream读数据写文件方式
* @param httpServletResponse
* @throws IOException
*/
public void streamDownload(HttpServletResponse httpServletResponse)
throws IOException {
AuthorsExample authorsExample = new AuthorsExample();
authorsExample.createCriteria();
HashMap<String, Object> param = new HashMap<>();
param.put("oredCriteria", authorsExample.getOredCriteria());
param.put("orderByClause", authorsExample.getOrderByClause());
CustomResultHandler customResultHandler = new CustomResultHandler(new DownloadProcessor (httpServletResponse));
sqlSessionTemplate.select(
"com.alphathur.mysqlstreamingexport.mapper.AuthorsMapper.streamByExample", param, customResultHandler);
httpServletResponse.getWriter().flush();
httpServletResponse.getWriter().close();
}
/**
* 传统下载方式
* @param httpServletResponse
* @throws IOException
*/
public void traditionDownload(HttpServletResponse httpServletResponse)
throws IOException {
AuthorsExample authorsExample = new AuthorsExample();
authorsExample.createCriteria();
List<Authors> authors = authorsMapper.selectByExample (authorsExample);
DownloadProcessor downloadProcessor = new DownloadProcessor (httpServletResponse);
authors.forEach (downloadProcessor::processData);
httpServletResponse.getWriter().flush();
httpServletResponse.getWriter().close();
}
}
controller
@RestController
@RequestMapping("download")
public class HelloController {
private final AuthorsService authorsService;
public HelloController(AuthorsService authorsService) {
this.authorsService = authorsService;
}
@GetMapping("streamDownload")
public void streamDownload(HttpServletResponse response)
throws IOException {
authorsService.streamDownload(response);
}
@GetMapping("traditionDownload")
public void traditionDownload(HttpServletResponse response)
throws IOException {
authorsService.traditionDownload (response);
}
}
Util工具类
public class DownloadProcessor {
private final HttpServletResponse response;
public DownloadProcessor(HttpServletResponse response) {
this.response = response;
String fileName = System.currentTimeMillis() + ".csv";
this.response.addHeader("Content-Type", "application/csv");
this.response.addHeader("Content-Disposition", "attachment; filename="+fileName);
this.response.setCharacterEncoding("UTF-8");
}
public <E> void processData(E record) {
try {
response.getWriter().write(record.toString()); //如果是要写入csv,需要重写toString,属性通过","分割
response.getWriter().write("\n");
}catch (IOException e){
e.printStackTrace();
}
}
}
写文件操作
自定义我们的ResultHandler，它用于获取java对象，然后传递给上面的DownloadProcessor处理类进行写文件操作
public class CustomResultHandler implements ResultHandler {
private final DownloadProcessor downloadProcessor;
public CustomResultHandler(
DownloadProcessor downloadProcessor) {
super();
this.downloadProcessor = downloadProcessor;
}
@Override
public void handleResult(ResultContext resultContext) {
Authors authors = (Authors)resultContext.getResultObject();
downloadProcessor.processData(authors);
}
}