背景 公司项目有个需求, 前端上传excel文件, 后端读取数据、处理数据、返回错误数据, 最简单的方式同步处理, 客户端上传文件后一直阻塞等待响应, 但用户体验无疑很差, 处理数据可能十分耗时, 没人愿意傻等, 由于项目暂未使用ActiveMQ等消息队列中间件, 而redis的lpush和rpop很适合作为一种轻量级的消息队列实现, 所以用它完成此次功能开发
一、本文涉及知识点
excel文件读写--阿里easyexcel sdk
文件上传、下载--腾讯云对象存储
远程服务调用--restTemplate
生产者、消费者--redisTemplate leftPush和rightPop操作
异步处理数据--Executors线程池
读取网络文件流--HttpClient
自定义注解实现用户身份认证--JWT token认证, 拦截器拦截标注有@LoginRequired注解的请求入口
当然, Java实现咯 涉及的知识点比较多, 每一个知识点都可以作为专题进行学习分析, 本文将完整实现呈现出来, 后期拆分与小伙伴分享学习
二、项目目录结构
项目结构
说明: 数据库DAO层放到另一个模块了, 不是本文重点
三、主要maven依赖
easyexcel
JWT
redis
腾讯cos
四、流程
用户上传文件
将文件存储到腾讯cos
将上传后的文件id及上传记录保存到数据库
redis生产一条导入消息, 即保存文件id到redis
请求结束, 返回"处理中"状态
redis消费消息
读取cos文件, 异步处理数据
将错误数据以excel形式上传至cos, 以供用户下载, 并更新处理状态为"处理完成"
客户端轮询查询处理状态, 并可以下载错误文件
结束
五、实现效果
上传文件
上传文件
数据库导入记录
数据库导入记录
导入的数据
导入的数据
下载错误文件
下载错误文件
错误数据提示
错误数据提示
查询导入记录
查询导入记录
六、代码实现
1、导入excel控制层
@LoginRequired@RequestMapping(value ="doImport", method = RequestMethod.POST)publicJsonResponse doImport(@RequestParam("file")MultipartFile file, HttpServletRequest request) { PLUser user = getUser(request);returnorderImportService.doImport(file, user.getId()); }
2、service层
@Override public JsonResponse doImport(MultipartFile file, Integer userId) {if(null == file || file.isEmpty()) { throw new ServiceException("文件不能为空"); } String filename = file.getOriginalFilename();if(!checkFileSuffix(filename)) { throw new ServiceException("当前仅支持xlsx格式的excel"); } // 存储文件 String fileId = saveToOss(file);if(StringUtils.isBlank(fileId)) { throw new ServiceException("文件上传失败, 请稍后重试"); } // 保存记录到数据库 saveRecordToDB(userId, fileId, filename); // 生产一条订单导入消息 redisProducer.produce(RedisKey.orderImportKey, fileId);returnJsonResponse.ok("导入成功, 处理中..."); } /** * 校验文件格式 * @param fileName * @return*/ private static boolean checkFileSuffix(String fileName) {if(StringUtils.isBlank(fileName) || fileName.lastIndexOf(".") <= 0) {returnfalse; } int pointIndex = fileName.lastIndexOf("."); String suffix = fileName.substring(pointIndex, fileName.length()).toLowerCase();if(".xlsx".equals(suffix)) {returntrue; }returnfalse; } /** * 将文件存储到腾讯OSS * @param file * @return*/ private String saveToOss(MultipartFile file) { InputStream ins = null; try { ins = file.getInputStream(); } catch (IOException e) { e.printStackTrace(); } String fileId; try { String originalFilename = file.getOriginalFilename(); File f = new File(originalFilename); inputStreamToFile(ins, f); FileSystemResource resource = new FileSystemResource(f); MultiValueMap param = new LinkedMultiValueMap<>(); param.add("file", resource); ResponseResult responseResult = restTemplate.postForObject(txOssUploadUrl, param, ResponseResult.class); fileId = (String) responseResult.getData(); } catch (Exception e) { fileId = null; }returnfileId; }
3、redis生产者
4、redis消费者
@ServicepublicclassRedisConsumer{@AutowiredpublicRedisTemplate redisTemplate;@Value("${txOssFileUrl}")privateString txOssFileUrl;@Value("${txOssUploadUrl}")privateString txOssUploadUrl;@PostConstructpublicvoidinit() { processOrderImport(); }/** * 处理订单导入 */privatevoid processOrderImport() { ExecutorService executorService = Executors.newCachedThreadPool(); executorService.execute(() -> {while(true) { Objectobject= redisTemplate.opsForList().rightPop(RedisKey.orderImportKey,1, TimeUnit.SECONDS);if(null==object) {continue; } String msg = JSON.toJSONString(object); executorService.execute(new OrderImportTask(msg, txOssFileUrl, txOssUploadUrl)); } }); }}
5、处理任务线程类
publicclassOrderImportTaskimplementsRunnable{publicOrderImportTask(String msg, String txOssFileUrl, String txOssUploadUrl){this.msg = msg;this.txOssFileUrl = txOssFileUrl;this.txOssUploadUrl = txOssUploadUrl; }}/** * 注入bean */privatevoidautowireBean(){this.restTemplate = BeanContext.getApplicationContext().getBean(RestTemplate.class);this.transactionTemplate = BeanContext.getApplicationContext().getBean(TransactionTemplate.class);this.orderImportService = BeanContext.getApplicationContext().getBean(OrderImportService.class); }@Overridepublicvoidrun(){// 注入bean autowireBean(); JSONObject jsonObject = JSON.parseObject(msg); String fileId = jsonObject.getString("fileId"); MultiValueMap
说明: 处理数据的业务逻辑代码就不用贴了
6、上传文件到cos
@RequestMapping("/txOssUpload") @ResponseBody public ResponseResult txOssUpload(@RequestParam("file") MultipartFile file) throws UnsupportedEncodingException {if(null== file || file.isEmpty()) {returnResponseResult.fail("文件不能为空"); } String originalFilename = file.getOriginalFilename(); originalFilename = MimeUtility.decodeText(originalFilename);//解决中文乱码问题 String contentType = getContentType(originalFilename); String key; InputStream ins =null; File f =null;try{ ins = file.getInputStream(); f =newFile(originalFilename); inputStreamToFile(ins, f); key = iFileStorageClient.txOssUpload(newFileInputStream(f), originalFilename, contentType); }catch(Exception e) {returnResponseResult.fail(e.getMessage()); }finally{if(null!= ins) {try{ ins.close(); }catch(IOException e) { e.printStackTrace(); } }if(f.exists()) {//删除临时文件 f.delete(); } }returnResponseResult.ok(key); } public static void inputStreamToFile(InputStream ins,File file) {try{ OutputStream os =newFileOutputStream(file); int bytesRead =0; byte[] buffer =newbyte[8192];while((bytesRead = ins.read(buffer,0,8192)) !=-1) { os.write(buffer,0, bytesRead); } os.close(); ins.close(); }catch(Exception e) { e.printStackTrace(); } } public String txOssUpload(FileInputStream inputStream, String key, String contentType) { key = Uuid.getUuid() +"-"+ key; OSSUtil.txOssUpload(inputStream, key, contentType);try{if(null!= inputStream) { inputStream.close(); } }catch(IOException e) { e.printStackTrace(); }returnkey; } public static void txOssUpload(FileInputStream inputStream, String key, String contentType) { ObjectMetadata objectMetadata =newObjectMetadata();try{ int length = inputStream.available(); objectMetadata.setContentLength(length); }catch(Exception e){ logger.info(e.getMessage()); } objectMetadata.setContentType(contentType); cosclient.putObject(txbucketName, key, inputStream, objectMetadata); }
7、下载文件
/** * 腾讯云文件下载 *@paramresponse *@paramid *@return*/@RequestMapping("/txOssDownload")publicObjecttxOssDownload(HttpServletResponse response, String id){ COSObjectInputStream cosObjectInputStream = iFileStorageClient.txOssDownload(id, response); String contentType = getContentType(id); FileUtil.txOssDownload(response, contentType, cosObjectInputStream, id);returnnull; }publicstaticvoidtxOssDownload(HttpServletResponse response, String contentType, InputStream fileStream, String fileName){ FileOutputStream fos =null; response.reset(); OutputStream os =null;try{ response.setContentType(contentType +"; charset=utf-8");if(!contentType.equals(PlConstans.FileContentType.image)){try{ response.setHeader("Content-Disposition","attachment; filename="+newString(fileName.getBytes("UTF-8"),"ISO8859-1")); }catch(UnsupportedEncodingException e) { response.setHeader("Content-Disposition","attachment; filename="+ fileName); logger.error("encoding file name failed", e); } } os = response.getOutputStream();byte[] b =newbyte[1024*1024];intlen;while((len = fileStream.read(b)) >0) { os.write(b,0, len); os.flush();try{if(fos !=null) { fos.write(b,0, len); fos.flush(); } }catch(Exception e) { logger.error(e.getMessage()); } } }catch(IOException e) { IOUtils.closeQuietly(fos); fos =null; }finally{ IOUtils.closeQuietly(os); IOUtils.closeQuietly(fileStream);if(fos !=null) { IOUtils.closeQuietly(fos); } } }
8、读取网络文件流
/** * 读取网络文件流 *@paramurl *@return*/publicstaticInputStream readFileFromURL(String url) {if(StringUtils.isBlank(url)) {returnnull; } HttpClient httpClient =newDefaultHttpClient(); HttpGet methodGet =newHttpGet(url);try{ HttpResponse response = httpClient.execute(methodGet);if(response.getStatusLine().getStatusCode() ==200) { HttpEntity entity = response.getEntity();returnentity.getContent(); } }catch(Exceptione) { e.printStackTrace(); }returnnull; }
9、ExcelUtil
/** * 读excel *@paraminputStream 文件输入流 *@returnlist集合 */publicstaticList read(InputStream inputStream) {returnEasyExcelFactory.read(inputStream,newSheet(1,1)); }/** * 写excel *@paramdata list数据 *@paramclazz *@paramsaveFilePath 文件保存路径 *@throwsIOException */publicstaticvoid write(List data,Classclazz,StringsaveFilePath)throwsIOException{ File tempFile =newFile(saveFilePath); OutputStream out =newFileOutputStream(tempFile); ExcelWriter writer = EasyExcelFactory.getWriter(out); Sheet sheet =newSheet(1,3, clazz,"Sheet1",null); writer.write(data, sheet); writer.finish(); out.close(); }
说明: 至此, 整个流程算是完整了, 下面将其他知识点代码也贴出来参考
七、其他
1、@LoginRequired注解
/** * 在需要登录验证的Controller的方法上使用此注解 */@Target({ElementType.METHOD})@Retention(RetentionPolicy.RUNTIME)public@interfaceLoginRequired {}
2、MyControllerAdvice
@ControllerAdvicepublicclassMyControllerAdvice{@ResponseBody@ExceptionHandler(TokenValidationException.class)publicJsonResponse tokenValidationExceptionHandler() {returnJsonResponse.loginInvalid(); }@ResponseBody@ExceptionHandler(ServiceException.class)publicJsonResponse serviceExceptionHandler(ServiceException se) {returnJsonResponse.fail(se.getMsg()); }@ResponseBody@ExceptionHandler(Exception.class)publicJsonResponse exceptionHandler(Exception e) { e.printStackTrace();returnJsonResponse.fail(e.getMessage()); }}
3、AuthenticationInterceptor
publicclassAuthenticationInterceptorimplementsHandlerInterceptor{privatestaticfinalString CURRENT_USER ="user";@AutowiredprivateUserService userService;@OverridepublicbooleanpreHandle(HttpServletRequest request, HttpServletResponse response, Object handler){// 如果不是映射到方法直接通过 if (!(handler instanceof HandlerMethod)) { return true; } HandlerMethod handlerMethod = (HandlerMethod) handler; Method method = handlerMethod.getMethod(); // 判断接口是否有@LoginRequired注解, 有则需要登录 LoginRequired methodAnnotation = method.getAnnotation(LoginRequired.class); if (methodAnnotation != null) { // 验证token Integer userId = JwtUtil.verifyToken(request); PLUser plUser = userService.selectByPrimaryKey(userId); if (null == plUser) { throw new RuntimeException("用户不存在，请重新登录"); } request.setAttribute(CURRENT_USER, plUser); return true; } return true; } @Override public void postHandle(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, ModelAndView modelAndView) throws Exception { } @Override public void afterCompletion(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, Exception e) throws Exception { }}
4、JwtUtil
public static final long EXPIRATION_TIME =2592_000_000L; // 有效期30天 public static final String SECRET ="pl_token_secret"; public static final String HEADER ="token"; public static final String USER_ID ="userId"; /** * 根据userId生成token * @param userId * @return*/ public static String generateToken(String userId) { HashMap map = new HashMap<>(); map.put(USER_ID, userId); String jwt = Jwts.builder() .setClaims(map) .setExpiration(new Date(System.currentTimeMillis() + EXPIRATION_TIME)) .signWith(SignatureAlgorithm.HS512, SECRET) .compact();returnjwt; } /** * 验证token * @param request * @return验证通过返回userId */ public static Integer verifyToken(HttpServletRequest request) { String token = request.getHeader(HEADER);if(token != null) { try { Map body = Jwts.parser() .setSigningKey(SECRET) .parseClaimsJws(token) .getBody();for(Map.Entry entry : body.entrySet()) { Object key = entry.getKey(); Object value = entry.getValue();if(key.toString().equals(USER_ID)) {returnInteger.valueOf(value.toString());// userId } }returnnull; } catch (Exception e) { logger.error(e.getMessage()); throw new TokenValidationException("unauthorized"); } }else{ throw new TokenValidationException("missing token"); } }