csv文件生成与解析

        
            com.alibaba.fastjson2
            fastjson2
            2.0.1
        

        
            org.apache.commons
            commons-csv
            1.9.0
        

1. csv文件生成

    // 接口地址
    @Value("${jk.device.uri}")
    private String requestURI;

    // 生成文件存放路径
    @Value("${jk.file.path}")
    private String filePath;


    @Scheduled(cron ="0 0 1 * * ?") // 每天凌晨1点执行
    public void requestDeviceData() {
//        System.out.println("定时任务执行了");

        DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd");
        //获取前一天日期
        LocalDate date = LocalDate.now().minusDays(1);
        String dateStr = date.format(fmt);

        String requestURL = requestURI + "?mdmTime=" + dateStr;

        RestTemplate restTemplate = new RestTemplate();

        JSONObject jsonObject =
                restTemplate.getForObject(requestURL, JSONObject.class);

        //判断有没有数据
        if (jsonObject == null || jsonObject.isEmpty()) {
            return;
        }
        JSONArray data = jsonObject.getJSONArray("data");

        if (data == null || data.isEmpty()) {
            return;
        }

        List list = data.toJavaList(JkOfflineDevice.class);
        if (list == null || list.isEmpty()) {
            return;
        }

        String fileName = "mdmData_" + dateStr + ".csv";

        //生成文件
        generateFile(list, filePath + fileName);
   

    }



    /**
     * 生成csv文件
     */
    public void  generateFile(List list, String fileName)   {

        try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName ));
             CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) {

            // 写入表头
            csvPrinter.printRecord(
                    "id",
                    "phoneno",
                    "sfzh",
                    "imei",
                    "imsi",
                    "manufacturer",
                    "product",
                    "statusDevice",
                    "createTime",
                    "mdmTime",
                    "area",
                    "statusPhone" );

            // 写入数据行
            for (JkOfflineDevice jd : list) {
                csvPrinter.printRecord(
                        jd.getId(),
                        jd.getPhoneno(),
                        jd.getSfzh(),
                        jd.getImei(),
                        jd.getImsi(),
                        jd.getManufacturer(),
                        jd.getProduct(),
                        jd.getStatusDevice(),
                        jd.getCreateTime(),
                        jd.getMdmTime(),
                        jd.getArea(),
                        jd.getStatusPhone()
                );
            }

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

2. 解析csv文件

    @Value("${jk.ftp.path}")
    private String filePath;

    private static final String [] headerName = {"id","phoneno","sfzh","imei","imsi","manufacturer","product","statusDevice","createTime","mdmTime","area","statusPhone"};


    public void parseFile() {

//        DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd");
//        //获取前一天日期
//        LocalDate date = LocalDate.now().minusDays(1);
//        String dateStr = date.format(fmt);
//
//        String fname = filePath + "mdmData_" +  dateStr + ".csv";
//
//        File file = new File(fname);
//
//        String fileName = file.getName();
//        long  time = file.lastModified();
//        System.out.println("文件名：" + fileName + "  时间：" + time);


        //   表记录最后更新时间， 按照这个时间读取之后的文件
        // 遍历每一个文件，比较时间，大于数据库记录时间的就解析保存，小于的掠过

        Integer recordId = 1;
//        Long  recordTime = recordMapper.getLastTimeById(recordId);
        Long  recordTime = 1L;

        // 如果数据库记录时间不存在（说明本次操作是第一次解析同步），就使用一个比较早的时间数据
        if (recordTime == null ){
            recordTime = 123456789L;
        }

        // 遍历目录下所有文件
        File directory = new File(filePath);
        if (!directory.exists() || !directory.isDirectory()) {
            directory.mkdirs();
        }

        File[] fileList = directory.listFiles();
        if (fileList == null){
            return;
        }
        for (File file : fileList){
            String fileName = file.getName();
            long  time = file.lastModified();

            // 不是csv文件，或者不是 mdmData_ 目标文件
            if (!fileName.contains(".csv") || !fileName.contains("mdmData_")){
                continue;
            }


            if (time <= recordTime){
                continue;
            }

            List list = parseCSV(file);

            if (list == null || list.isEmpty()){
                continue;
            }

            System.out.println("文件名：" + fileName + "  time：" + time + "时间："+ LocalDateTime.now());

            System.out.println("解析完成：" + list.size());
            System.out.println(list.get(0).toString());

        }

    }


    /**
     * 把指定格式的CVS文件解析成JkOfflineDevice对象集合
     * @param file
     * @return
     */
    private List parseCSV(File file){

        List list = null;

        if (file == null || !file.exists()){
            return list;
        }

        try (CSVParser parser = CSVParser.parse(
                new FileReader(file),
                CSVFormat.DEFAULT.withFirstRecordAsHeader())) {

            //表头格式校验
            // id,phoneno,sfzh,imei,imsi,manufacturer,product,statusDevice,createTime,mdmTime,area,statusPhone
            List headerList = parser.getHeaderNames();
            boolean isValid = new HashSet<>(headerList).containsAll(Arrays.asList(headerName));

            //表头不是预期格式,说明该文件不是目标文件，跳过
            if (!isValid) {
                //System.out.println("文件格式错误，跳过");
                return list;
            }

           // System.out.println("文件格式正确！！");

            list = new ArrayList<>();

            for (CSVRecord record : parser) {

                JkOfflineDevice jd = new JkOfflineDevice();

                jd.setId(record.get("id"));
                jd.setPhoneno(record.get("phoneno"));
                jd.setSfzh(record.get("sfzh"));
                jd.setImei(record.get("imei"));
                jd.setImsi(record.get("imsi"));
                jd.setManufacturer(record.get("manufacturer"));
                jd.setProduct(record.get("product"));
                jd.setStatusDevice(Integer.valueOf(record.get("statusDevice")));
                jd.setCreateTime(record.get("createTime"));
                jd.setMdmTime(record.get("mdmTime"));
                jd.setArea(record.get("area"));
                jd.setStatusPhone(record.get("statusPhone"));

                list.add(jd);
            }
        } catch (IOException e) {
            list = null;
            e.printStackTrace();
        }

        return list;
    }