长时间页面请求的后台处理
主要设计思路：
接收到请求，对请求计算，将任务加入简易任务池，反馈给请求方任务的总体情况；
再由其他进程执行具体的耗时任务；
任务执行过程中，更新任务池中任务进度；
系统维护一个websocket，专门用于接收状态监控，实时返回任务处理状态；
任务执行时向websocket的任务订阅者发送最新进度；
提供接口查询任务池中的任务状态。
主要类：
SyncTask，任务Bean定义
MyProgressWebSocket，websocket接收用户请求，后向用户反馈任务信息
@Api(tags = "ProgressWebSocket")
@Component
@ServerEndpoint(value="/ws/progress/{taskId}")
public class ProgressWebSocket extends AbstractProgressWebSocket {
}
public abstract class AbstractProgressWebSocket extends AbstractWebSocket {
private static final Logger log = LoggerFactory.getLogger(AbstractProgressWebSocket.class);
public AbstractProgressWebSocket() {
}
public void linkTransport() {
AbstractProgressWebSocket.Transport.setSocketHandler(this);
}
public static class Transport {
private static AbstractWebSocket parent = null;
public Transport() {
}
private static void setSocketHandler(AbstractWebSocket webSocketHander) {
parent = webSocketHander;
}
public static void sendMessage(TaskProgress progress) {
if (parent != null) {
parent.sendMessage(progress.getTaskId(), JSONObject.toJSONString(progress));
}
}
}
}
import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import javax.websocket.OnClose;
import javax.websocket.OnError;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.Session;
import javax.websocket.server.PathParam;
import org.apache.commons.lang3.ObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class AbstractWebSocket {
private static final Logger log = LoggerFactory.getLogger(AbstractWebSocket.class);
private static ConcurrentHashMap<Session, AbstractWebSocket> webSocketMap = new ConcurrentHashMap();
private static ConcurrentHashMap<String, Session> workSessionMap = new ConcurrentHashMap();
public AbstractWebSocket() {
}
protected abstract void linkTransport();
@OnOpen
public final void onOpen(Session session, @PathParam("taskId") String taskId) {
webSocketMap.put(session, this);
workSessionMap.put(taskId, session);
this.linkTransport();
this.sendMessage(session, "task:" + taskId);
}
@OnClose
public final void onClose(Session closeSession) {
webSocketMap.remove(closeSession);
Iterator var2 = workSessionMap.keySet().iterator();
while(var2.hasNext()) {
String workId = (String)var2.next();
if (ObjectUtils.isNotEmpty(workId) && ObjectUtils.isNotEmpty(workSessionMap.get(workId)) && ((Session)workSessionMap.get(workId)).equals(closeSession)) {
workSessionMap.remove(workId);
}
}
}
@OnError
public final void onError(Session session, Throwable error) {
}
@OnMessage
public final void onMessage(String workId, Session mySession) {
if (ObjectUtils.isNotEmpty(workId)) {
Iterator var3 = webSocketMap.keySet().iterator();
while(var3.hasNext()) {
Session session = (Session)var3.next();
if (session.equals(mySession)) {
workSessionMap.put(workId, session);
String msg = "receive subscribe for " + workId;
this.sendMessage(mySession, msg);
}
}
}
}
private void sendMessage(Session mySession, String message) {
try {
if (mySession.isOpen()) {
if (message != null) {
mySession.getBasicRemote().sendText(message);
}
}
} catch (IOException var4) {
log.error("websocker error");
}
}
protected void sendMessage(String taskId, String msg) {
if (webSocketMap != null) {
if (workSessionMap.containsKey(taskId)) {
this.sendMessage((Session)workSessionMap.get(taskId), msg);
}
}
}
}
大致思路：
使用简易任务队列，任务仅存放内存，这些任务都是即时性的，如果服务关闭，任务终止，所以任务列表使用变量记录就可以；任务进度会持久化到数据库；
请求的处理：
接收到大任务请求；
创建任务，放入任务队列缓存，持久化任务信息；
新的线程执行任务；
立即返回任务信息描述，如任务处理的数据量。
任务处理：
更新任务的处理进度；
更新持久化任务的进度；
尝试向websocket订阅者发送最新进度；
任务处理完毕，最终更新任务状态，更新持久化任务状态；发布到websocket。
SyncTask定义：
@Data
@ApiModel(value = "异步任务表")
public class SyncTask {
@ApiModelProperty(value = "ID")
private Long id;
@ApiModelProperty(value = "批处理任务ID")
private String taskId;
@ApiModelProperty(value = "任务类型")
private String taskType;
@ApiModelProperty(value = "任务名称")
private String taskName;
@ApiModelProperty(value = "处理索引")
private Integer index;
@ApiModelProperty(value = "处理总数")
private Integer total;
@ApiModelProperty(value = "历时毫秒数")
private Long lapse;
@ApiModelProperty(value = "开始时间")
private Date startTime;
@ApiModelProperty(value = "结束时间")
private Date endTime;
@ApiModelProperty(value = "批处理成功数")
private Integer success;
@ApiModelProperty(value = "批处理失败数")
private Integer failure;
@ApiModelProperty(value = "更多信息")
@Column(length = 200)
private String moreDetailUrl;
}
辅助类
public interface TaskProgressCallback {
void taskGoing(SyncTask task, int index);
}
任务处理的类
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.RandomUtils;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.util.*;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@Slf4j
public class FileServiceImpl implements TaskProgressCallback {
@Resource
private TaskRepository taskRepository;
@Resource
private ThreadPoolExecutor executor;
private AtomicInteger atomicInteger = new AtomicInteger(0);
private Map<String, SyncTask> myTaskList = new HashMap<>();
/**
* 占用时间可能比较长的函数
*/
public void receiveBigZipFile(MultipartFile file) {
List<File> files = getFilesInZip(file);
log.info("文件数量：{}" ,files.size());
int total = files.size();
//1、创建任务，放入列表
SyncTask task = createTask(null, "face.import", total);
String taskId = task.getTaskId();
myTaskList.put(taskId,task);
executor.execute(() -> {
List<File> errorFiles = new ArrayList<>();
//2、依次循环处理faceFiles中的图片
BatchOperationResult excuteResult = doTask(files, errorFiles, taskId);
if(errorFiles.size()>0){
//处理加工执行结果
String suffix = ".zip";
File errorZipFile = File.createTempFile( "error.",suffix);
zipFiles(errorFiles,errorZipFile);
//压缩包放入ceph
String errorUrl = uploadTempFileAndReturnFileUrl(errorZipFile,suffix,"error");
//返回错误文件完整路径
excuteResult.setErrorFilePath(errorUrl);
}
//记录任务状态，只变量缓存即可
if(ObjectUtils.isNotEmpty(taskId)) {
//更新列表
myTaskList.put(taskId,task);
//记录任务状态
finishTask(task,excuteResult.getSuccess(), excuteResult.getFailure(), excuteResult.getErrorFilePath());
//发送消息
MyProgressWS.Transport.sendMessage(task);
}
});
}
/**
* 创建任务ID
* @return 任务id，
*/
private synchronized String generateTaskId() {
String taskId = null;
synchronized (atomicInteger) {
taskId = atomicInteger.addAndGet(1) + "-" + System.nanoTime() ;
}
return taskId;
}
/**
* 创建任务
* @param taskName 任务名称，可前端命名
* @param taskType 任务类型，非必填
* @param total 任务要处理的总数
* @return SyncTask
*/
private SyncTask createTask(String taskName, String taskType, int total){
String taskId = generateTaskId();
if(ObjectUtils.isEmpty(taskName)){
taskName = taskId;
}
SyncTask task = new SyncTask();
task.setTaskName(taskName);
task.setTaskType(taskType);
task.setTaskId(taskId);
task.setStartTime(Calendar.getInstance().getTime());
task.setTotal(total);
taskRepository.save(task);
return task;
}
/**
* 标记任务结束，并记录
* @param task 任务
* @param succNum 最终成功数
* @param failNum 最终失败数
* @param filePathOrUrl 失败文件，或者成功文件
*/
private void finishTask(SyncTask task,int succNum,int failNum, String filePathOrUrl){
task.setSuccess(succNum);
task.setEndTime(Calendar.getInstance().getTime());
task.setFailure(failNum);
task.setErrorFilePath(filePathOrUrl);
taskRepository.save(task);
}
/**
* 定向广播发送进度
* @param task
* @param index
*/
public void taskGoing(TaskProgress task, int index){
if(ObjectUtils.isNotEmpty(task)){
task.setIndex(index);
if(ObjectUtils.isNotEmpty(task.getStartTime())) {
task.setLapse(System.currentTimeMillis() - task.getStartTime().getTime());
}
//发送消息
MyProgressWS.Transport.sendMessage(task);
}
}
/**
* @return 失败和成功个数
*/
private BatchOperationResult doTask(List<File> fileList, List<File> outErrorFiles, String taskId){
int succNum = 0;
int failNum =0;
Integer fileIndex = 0;
long startTime = System.currentTimeMillis();
int totalFileCount = fileList.size();
int step = Integer.parseInt(""+ (1 + totalFileCount / 50));
for(File file : fileList){
fileIndex++;
//do something that need a mount of time
boolean doSomethingResult = file.exists() && RandomUtils.nextInt()>5;
if(doSomethingResult)
succNum++;
else
failNum++;
//记录任务状态，只变量缓存即可
if(ObjectUtils.isNotEmpty(taskId)) {
SyncTask syncTask = myTaskList.getOrDefault(taskId,new SyncTask());
syncTask.setLapse(System.currentTimeMillis()-startTime);
syncTask.setIndex(fileIndex);
syncTask.setTotal(totalFileCount);
if(fileIndex == totalFileCount){
syncTask.setEndTime(Calendar.getInstance().getTime());
}
myTaskList.put(taskId,syncTask);
MyProgressWS.Transport.sendMessage(syncTask);
}
}
BatchOperationResult excuteResult = new BatchOperationResult(succNum,failNum);
return excuteResult;
}
}