远程xml读取解析，将image url下载到本地，延时队列定时删除文件，图片访问路径保存在数据库中

远程xml部分内容

"FY4A AGRI IMG REGI MTCC GLL" tag="FY4A AGRI IMG REGI MTCC GLL">
	"2023-07-25 22:30 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725223000_20230725223417_1000M_V0001.JPG"/>
	"2023-07-25 22:23 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725222336_20230725222753_1000M_V0001.JPG"/>

mq发布端定时任务发送消息

@Component
public class TFYImage {
    private ISyncFY syncFY;
    Log log = LogFactory.getLog(TFYImage.class);
    @Autowired
    public TFYImage(ISyncFY syncFY){
        this.syncFY = syncFY;
    }

    @PostConstruct
    @Scheduled(cron = "0 0 * * * *")
    public void Task1() {
        log.info("execute FY Image task");
        String xmlUrl = "xml path";
        String localPath = "/media/resource/FY4APic/";
        String type = "FY4A";
        syncFY.syncFY4AImage(xmlUrl, localPath, type);
    }
}

mq消费端
1，远程xml读取
2，xml解析，将image中图片url保存在集合中
3，遍历集合，当本地不存在此图片时，下载图片至本地
4，将图片路径传给延时队列，用于稍后删除图片
5，保存自定义图片访问路径等信息到数据库

import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.MQMessage;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.xugu1.fy.IFYImageMapper;
import com.sxqx.pojo.FYImageItem;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.file.FileHelper;
import com.sxqx.utils.mq.MQMessageSender;
import com.sxqx.utils.xml.XMLUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;

@Component
public class SyncFYImgReceiver implements IMessageReceiver {

    private final IFYImageMapper fyImageMapper;
    private final MQMessageSender mqMessageSender;

    @Autowired
    public SyncFYImgReceiver(MQMessageSender mqMessageSender,
                             IFYImageMapper fyImageMapper
    ) {
        this.mqMessageSender = mqMessageSender;
        this.fyImageMapper = fyImageMapper;
    }

    Log log = LogFactory.getLog(SyncFYImgReceiver.class);

    @RabbitListener(queuesToDeclare = {
            @Queue(name = "sxqxgxw_sync_fy_img")
    })
    @RabbitHandler
    @Override
    public void onMessageReceived(String mqMessageString) {
        JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
        JsonNode msg = jsonNode.findValue("msg");
        JsonNode JsonNodeParams = msg.findValue("params");
        Map, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
        if (params.size() > 0) {
            String xmlUrl = params.get("xmlUrl").toString();
            String localPath = params.get("localPath").toString();
            String type = params.get("type").toString();

            if(Objects.equals(type,"FY4A")){
                List imageUrlList = new ArrayList<>();
                try {
                    //读取xml
                    Document document = XMLUtil.readXMLUrl(xmlUrl);

                    // 创建一个XPath对象
                    XPathFactory xPathFactory = XPathFactory.newInstance();
                    XPath xPath = xPathFactory.newXPath();

                    // 使用XPath表达式获取imagelist节点
                    XPathExpression expr = xPath.compile("/imagelist");
                    NodeList nodeList = (NodeList) expr.evaluate(document, XPathConstants.NODESET);

                    // 遍历imagelist节点
                    Node node = nodeList.item(0);

                    if (node.getNodeType() == Node.ELEMENT_NODE) {
                        Element imagelistElement = (Element) node;

                        // 获取name和tag属性值
                        //String name = imagelistElement.getAttribute("name");
                        //String tag = imagelistElement.getAttribute("tag");
                        //System.out.println("name: " + name);
                        //System.out.println("tag: " + tag);

                        // 获取image节点列表
                        NodeList imageNodes = imagelistElement.getElementsByTagName("image");

                        // 遍历image节点
                        //读取xml中image存入imageUrlList
                        for (int j = 0; j < imageNodes.getLength(); j++) {
                            Node imageNode = imageNodes.item(j);

                            if (imageNode.getNodeType() == Node.ELEMENT_NODE) {
                                Element imageElement = (Element) imageNode;

                                // 获取image节点的time、desc和url属性值
                                //String time = imageElement.getAttribute("time");
                                //String desc = imageElement.getAttribute("desc");
                                String imageUrl = imageElement.getAttribute("url");
                                imageUrlList.add(imageUrl);
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }

                List fyImageItems = new ArrayList<>();

                //下载图片，组装数据
                for (String imageUrl : imageUrlList) {
                    // 获取最后一个斜杠的位置
                    int slashIndex = imageUrl.lastIndexOf("/");
                    // 截取文件名
                    String fileName = imageUrl.substring(slashIndex + 1);
                    String filePath = localPath;
                    String savePath = filePath + fileName;
                    File file = new File(savePath);
                    if (file.exists()) {
                        continue;
                    }
                    //下载图片
                    try {
                        long start = System.currentTimeMillis();
                        FileHelper.downloadUsingNIO(imageUrl, savePath);
                        long end = System.currentTimeMillis();
                        long timeDifferenceInSeconds = (end - start) / 1000;
                        log.info(savePath + " 下载耗时：" + timeDifferenceInSeconds + "秒");
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                    if (file.exists() && file.length() > 0) {
                        //组装数据
                        String[] s = fileName.split("_");
                        if (s.length == 13) {
                            try {
                                String product = s[0].replace("-", "");
                                String dt = s[9].substring(0, 14);
                                SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
                                Date date = sdf.parse(dt);
                                Calendar bjc = Calendar.getInstance();
                                bjc.setTime(date);
                                bjc.add(Calendar.HOUR_OF_DAY, 8);
                                date = bjc.getTime();
                                FYImageItem fyImageItem = new FYImageItem();
                                fyImageItem.setTime(date);
                                fyImageItem.setPath("http://本地服务器/mediaResource/FY4APic/" + fileName);
                                fyImageItem.setFilename(fileName);
                                fyImageItem.setProduct(product);
                                fyImageItems.add(fyImageItem);
                            } catch (Exception e) {
                                e.printStackTrace();
                            }
                        }
                    }

                    // 清除数据
                    MQMessage mqMessage = new MQMessage();
                    JSONObject jsonObject = new JSONObject();
                    jsonObject.put("filePath", savePath);
                    mqMessage.setMsg(jsonObject);
                    mqMessageSender.send("queue.file_delay_destroy", mqMessage);
                }

                //保存到数据库
                if (fyImageItems.size() > 0) {
                    for (List fyImageItems1 : ListUtils.splitList(fyImageItems, 50)) {
                        int rows = fyImageMapper.insertIntoFYImage(fyImageItems1);
                        try {
                            Thread.sleep(50);
                        } catch (InterruptedException ex) {
                            ex.printStackTrace();
                        }
                    }
                    try {
                        Thread.sleep(100);
                    } catch (Exception e) {
                    }
                } else {
                    log.info("don't need to sync fy4a file and fyImageItems.size() is " + fyImageItems.size());
                }
            }
        }
    }
}

XMLUtil读取远程xml链接

import org.w3c.dom.Document;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.net.HttpURLConnection;
import java.net.URL;

public class XMLUtil {
    public static Document readXMLUrl(String xmlUrl) throws Exception{
        // 创建一个URL对象
        URL url = new URL(xmlUrl);
        // 创建一个DocumentBuilderFactory对象
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        // 创建一个DocumentBuilder对象
        DocumentBuilder builder = factory.newDocumentBuilder();
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        // 设置连接超时时间为5秒
        connection.setConnectTimeout(5000);
        // 设置读取超时时间为10秒
        connection.setReadTimeout(10000);
        // 从URL中读取XML文件并解析
        Document document = builder.parse(connection.getInputStream());
        return document;
    }
}

使用NIO下载文件

package com.sxqx.utils.file;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;

public class FileHelper {
    public static void deleteFile(File file) {
        /*
          File[] listFiles()
           返回一个抽象路径名数组，这些路径名表示此抽象路径名表示的目录中的文件。
         */

        File[] files = file.listFiles();
        if (files!=null) {//如果包含文件进行删除操作
            for (File value : files) {
                if (value.isFile()) {
                    //删除子文件
                    value.delete();
                } else if (value.isDirectory()) {
                    //通过递归的方法找到子目录的文件
                    deleteFile(value);
                }
                value.delete();//删除子目录
            }
        }
        file.delete();
    }

    public static Boolean downloadFile(String urlString, String savePath) {
        InputStream is = null;
        FileOutputStream os = null;
        try {
            // 构造URL
            URL url = new URL(urlString);
            // 打开连接
            URLConnection con = url.openConnection();
            // 输入流
            is = con.getInputStream();
            // 1K的数据缓冲
            byte[] bs = new byte[1024];
            // 读取到的数据长度
            int len;
            // 输出的文件流

            File file = new File(savePath);
            os = new FileOutputStream(file, true);
            // 开始读取
            while ((len = is.read(bs)) != -1) {
                os.write(bs, 0, len);
            }

            return true;
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        } finally {
            // 完毕，关闭所有链接
            try {
                if (null != os) {
                    os.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            try {
                if (null != is) {
                    is.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        //从微信下载图片时如果没有id对应的图片则下载一个空图片，不会存在返回为null的情况
    }

    /**
     * 使用NIO下载文件
     * @param urlStr
     * @param file
     * @throws IOException
     */
    public static void downloadUsingNIO(String urlStr, String file) throws IOException {
        URL url = new URL(urlStr);
        ReadableByteChannel rbc = Channels.newChannel(url.openStream());
        FileOutputStream fos = new FileOutputStream(file);
        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
        fos.close();
        rbc.close();
    }
}