springboot-实时监听FTP文件变化

概述

实现实时监听FTP服务器文件的新增、修改和删除，方便触发自定义业务，亲测可用。

引入依赖

   <dependency>
        <groupId>commons-netgroupId>
        <artifactId>commons-netartifactId>
        <version>3.6version>
    dependency>

FTP客户端（加强版）

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.net.ftp.*;

import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
import java.util.concurrent.ExecutorService;

@Slf4j
public class FTPClientPlus {

    private final String CONTROL_ENCODING = "UTF-8";

    private FTPClient client = null;

    private String host = "";
    private int port = 21;
    private String user = "";
    private String password = "";
    private String ftpPath = "/";

    public FTPClient getClient() {
        return client;
    }

    public void setClient(FTPClient client) {
        this.client = client;
    }

    @SuppressWarnings("unused")
    private FTPClientPlus() {
    }

    public FTPClientPlus(String host, int port, String user, String pwd) {
        this.host = host;
        this.port = port;
        this.user = user;
        this.password = pwd;
    }

    /**
     * 获取当前FTP所在目录位置
     *
     * @return
     */
    public String getHome() {
        return this.ftpPath;
    }

    /**
     * 连接FTP Server
     *
     * @throws IOException
     */
    public static final String ANONYMOUS_USER = "anonymous";

    public void connect() throws Exception {
        if (client == null) {
            client = new FTPClient();
        }
        // 已经连接
        if (client.isConnected()) {
            return;
        }

        // 编码
        client.setControlEncoding(CONTROL_ENCODING);

        try {
            // 连接FTP Server
            client.connect(this.host, this.port);
            // 登陆
            if (this.user == null || "".equals(this.user)) {
                // 使用匿名登陆
                client.login(ANONYMOUS_USER, ANONYMOUS_USER);
            } else {
                client.login(this.user, this.password);
            }
            // 设置文件格式
            client.setFileType(FTPClient.BINARY_FILE_TYPE);
            // 获取FTP Server 应答
            int reply = client.getReplyCode();
            if (!FTPReply.isPositiveCompletion(reply)) {
                client.disconnect();
                throw new Exception("connection FTP fail!");
            }

            FTPClientConfig config = new FTPClientConfig(client.getSystemType().split(" ")[0]);
            config.setServerLanguageCode("zh");
            client.configure(config);
            // 使用被动模式设为默认
            client.enterLocalPassiveMode();
            // 二进制文件支持
            client.setFileType(FTP.BINARY_FILE_TYPE);
            // 设置模式
            client.setFileTransferMode(FTP.STREAM_TRANSFER_MODE);

        } catch (IOException e) {
            throw new Exception("connection FTP fail! " + e);
        }
    }

    /**
     * 断开FTP连接
     *
     * @throws IOException
     */
    public void close() {
        if (client != null && client.isConnected()) {
            try {
                client.logout();
                client.disconnect();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 获取文件列表
     *
     * @return
     */
    public List<FTPFile> list() {
        List<FTPFile> list = null;
        try {
            FTPFile[] ff = client.listFiles(ftpPath);
            if (ff != null && ff.length > 0) {
                list = new ArrayList<FTPFile>(ff.length);
                Collections.addAll(list, ff);
            } else {
                list = new ArrayList<FTPFile>(0);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }

    /**
     * 切换目录
     *
     * @param path           需要切换的目录
     * @param forcedIncrease 如果目录不存在，是否增加
     */
    public void switchDirectory(String path, boolean forcedIncrease) {
        try {
            if (path != null && !"".equals(path)) {
                boolean ok = client.changeWorkingDirectory(path);
                if (ok) {
                    this.ftpPath = path;
                } else if (forcedIncrease) {
                    // ftpPath 不存在，手动创建目录
                    StringTokenizer token = new StringTokenizer(path, "\\//");
                    while (token.hasMoreTokens()) {
                        String npath = token.nextToken();
                        client.makeDirectory(npath);
                        client.changeWorkingDirectory(npath);
                        if (ok) {
                            this.ftpPath = path;
                        }
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建目录
     *
     * @param path
     */
    public void createDirectory(String path) {
        try {
            if (path != null && !"".equals(path)) {
                boolean ok = client.changeWorkingDirectory(path);
                if (!ok) {
                    // ftpPath 不存在，手动创建目录
                    StringTokenizer token = new StringTokenizer(path, "\\//");
                    while (token.hasMoreTokens()) {
                        String npath = token.nextToken();
                        client.makeDirectory(npath);
                        client.changeWorkingDirectory(npath);
                    }
                }
            }
            client.changeWorkingDirectory(this.ftpPath);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 删除目录，如果目录中存在文件或者文件夹则删除失败
     *
     * @param path
     * @return
     */
    public boolean deleteDirectory(String path) {
        boolean flag = false;
        try {
            flag = client.removeDirectory(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flag;
    }

    /**
     * 删除文件
     *
     * @param path
     * @return
     */
    public boolean deleteFile(String path) {
        boolean flag = false;
        try {
            flag = client.deleteFile(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flag;
    }

    /**
     * 上传文件，上传文件只会传到当前所在目录
     *
     * @param localFile 本地文件
     * @return
     */
    public boolean upload(File localFile) {
        return this.upload(localFile, "");
    }

    /**
     * 上传文件，会覆盖FTP上原有文件
     *
     * @param localFile 本地文件
     * @param reName    重名
     * @return
     */
    public boolean upload(File localFile, String reName) {
        boolean flag = false;
        String targetName = reName;
        // 设置上传后文件名
        if (reName == null || "".equals(reName)) {
            targetName = localFile.getName();
        }
        FileInputStream fis = null;
        try {
            // 开始上传文件
            fis = new FileInputStream(localFile);
            client.setControlEncoding(CONTROL_ENCODING);
            client.setFileType(FTPClient.BINARY_FILE_TYPE);
            boolean ok = client.storeFile(targetName, fis);
            if (ok) {
                flag = true;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return flag;
    }

    /**
     * 下载文件，如果存在会覆盖原文件
     *
     * @param ftpFileName 文件名称，FTP上的文件名称
     * @param savePath    保存目录，本地保存目录
     * @return
     */
    public boolean download(String ftpFileName, String savePath) throws IOException {
        log.info("开始下载 FTP:{} 文件：{}", client.getPassiveHost(), ftpFileName);
        boolean flag = false;

        File dir = new File(savePath);

        if (!dir.exists()) {
            dir.mkdirs();
        }

        FileOutputStream fos = null;
            String saveFile = dir.getAbsolutePath() + File.separator + ftpFileName;
            fos = new FileOutputStream(saveFile);
            boolean ok = client.retrieveFile(ftpFileName, fos);
            if (ok) {
                flag = true;
                log.info("开始下载 FTP:{} 文件：{} 成功保存到：{}", client.getPassiveHost(), ftpFileName, savePath);
            }
        return flag;
    }


    public void addListenerFileChange(FileChangeEvent fileChangeEvent, ExecutorService executorService) {
        ListenerFileChangeThreadRunnable listenerFileChangeThread = new ListenerFileChangeThreadRunnable(client,
                fileChangeEvent);
        executorService.submit(listenerFileChangeThread);
//        new Thread(listenerFileChangeThread).start();
    }

}

触发数据类

import lombok.Data;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPFile;

import java.io.IOException;
import java.io.InputStream;
import java.util.Objects;

@Data
public class FileChangeData {

    /**
     * 文件信息
     */
    private FTPFile ftpFile;

    /**
     * 文件改变类型
     */
    private FileChangeType eventType;

    /**
     * 文件名称
     */
    private String fileName;

    /**
     * 文件大小
     */
    private Long fileSize;

    /**
     * FTPClient
     */
    private FTPClient ftpClient;

    /**
     * 获取文件输入流
     *
     * @return InputStream
     */
    public InputStream getInputStream() {
        //如果是删除事件则不能够获取流
        if (Objects.equals(eventType, FileChangeType.FILE_DELETED)) {
            return null;
        }

        try {
            return ftpClient.retrieveFileStream(this.fileName);
        } catch (IOException e) {
            return null;
        }
    }
}

触发事件接口（函数式）

@FunctionalInterface
public interface FileChangeEvent {

    /**
     * 文件发生改变时触发此方法
     * @param fileChangeData 文件发生了改变
     * */
    void change(FileChangeData fileChangeData);

}

触发事件类型

public enum FileChangeType {
    FILE_UPDATE(0, "文件更新"),
    FILE_ADD(1, "文件添加"),
    FILE_DELETED(2, "文件删除");

    @Getter
    private Integer type;

    @Getter
    private String desc;

    FileChangeType(Integer type, String desc) {
        this.type = type;
        this.desc = desc;
    }
}

处理线程任务

@Slf4j
public class ListenerFileChangeThreadRunnable implements Runnable {

    private final FTPClient ftpClient;

    private volatile boolean stop;

    private final Map<String, Long> fileMemory;

    private final String fileMemoryKey;

    private final FileChangeEvent fileChangeEvent;

    public ListenerFileChangeThreadRunnable(FTPClient ftpClient, FileChangeEvent fileChangeEvent) {
        this.ftpClient = ftpClient;
        this.fileChangeEvent = fileChangeEvent;
        this.fileMemoryKey = FTPChangeConstants.ChannelMapping_Cache_Key_Prefix + ftpClient.getPassiveHost();
        //读取上次的缓存
        if (RedisUtils.hasKey(fileMemoryKey) && ObjectUtil.isNotEmpty(RedisUtils.getCacheMap(fileMemoryKey))) {
            //TODO:后期要考虑缓存数据量大 影响性能的问题
            this.fileMemory = RedisUtils.getCacheMap(fileMemoryKey);
        } else {
            this.fileMemory = new HashMap<>();
        }
    }

    @Override
    public void run() {
        while (!stop) {
            try {
                FTPFile[] ftpFiles = ftpClient.listFiles();

                //判断文件被删除
                hanleDel(ftpFiles);

                //判断文件是否有更改或新增
                for (FTPFile ftpFile : ftpFiles) {
                    //判断是否为文件夹
                    if (ftpFile.isDirectory()) {
//                        log.info("{}为文件不进行监听操作", ftpFile.getName());
                        continue;
                    }
                    FileChangeData fileChangeData = new FileChangeData();
                    fileChangeData.setFileName(ftpFile.getName());
                    fileChangeData.setFileSize(ftpFile.getSize());
                    fileChangeData.setFtpFile(ftpFile);
                    fileChangeData.setFtpClient(ftpClient);
                    //文件是否存在于缓存文件列表中
                    if (fileMemory.containsKey(ftpFile.getName())) {
//                        log.info("文件{}在内存中已经存在，进行大小判断", ftpFile.getName());
                        handleUpdate(ftpFile, fileChangeData);
                        continue;
                    }
                    handleAdd(ftpFile, fileChangeData);
                }
            } catch (Exception e) {
                log.error("FTP监控异常：{}", e);
                throw new RuntimeException(e);
            }
            try {
                TimeUnit.SECONDS.sleep(5);
            } catch (InterruptedException e) {
                log.error("FTP监控异常：{}", e);
                throw new RuntimeException(e);
            }
        }
    }

    private void handleAdd(FTPFile ftpFile, FileChangeData fileChangeData) {
        //log.info("文件{}在内存中不存在进行缓存操作", ftpFile.getName());
        log.info("监听到FTP: {} 新增文件：{}", ftpClient.getPassiveHost(), ftpFile.getName());
        fileMemory.put(ftpFile.getName(), ftpFile.getSize());
        RedisUtils.setCacheMapValue(fileMemoryKey, ftpFile.getName(), ftpFile.getSize());
        fileChangeData.setEventType(FileChangeType.FILE_ADD);
        fileChangeEvent.change(fileChangeData);
    }

    private void handleUpdate(FTPFile ftpFile, FileChangeData fileChangeData) {
        if (!Objects.equals(fileMemory.get(ftpFile.getName()), ftpFile.getSize())) {
//                            log.info("文件{}在内存中已经存在且大小不一致，进行更新缓存操作", ftpFile.getName());
            log.info("监听到FTP: {} 更新文件：{}", ftpClient.getPassiveHost(), ftpFile.getName());

            fileMemory.put(ftpFile.getName(), ftpFile.getSize());
            RedisUtils.setCacheMapValue(fileMemoryKey, ftpFile.getName(), ftpFile.getSize());

            fileChangeData.setEventType(FileChangeType.FILE_UPDATE);
            fileChangeEvent.change(fileChangeData);
        }
    }

    private void hanleDel(FTPFile[] ftpFiles) {
        if (fileMemory.size() > 0) {
            Set<String> fileNames = new HashSet<>();
            for (FTPFile ftpFile : ftpFiles) {
                if (ftpFile.isDirectory()) {
//                            log.info("文件夹:{} 不做删除判断",ftpFile.getName());
                    continue;
                }
                fileNames.add(ftpFile.getName());
            }
            Set<Map.Entry<String, Long>> entries = fileMemory.entrySet();
            List<String> removeKeys = new ArrayList<>();
            for (Map.Entry<String, Long> entry : entries) {
                if (!fileNames.contains(entry.getKey())) {
                    log.info("监听到FTP: {} 删除文件：{}", ftpClient.getPassiveHost(), entry.getKey());
                    FileChangeData fileChangeData = new FileChangeData();
                    fileChangeData.setEventType(FileChangeType.FILE_DELETED);
                    fileChangeData.setFileName(entry.getKey());
                    fileChangeData.setFileSize(entry.getValue());
                    fileChangeEvent.change(fileChangeData);
                    removeKeys.add(entry.getKey());
                }
            }

            removeKeys.forEach(rmkey -> {
                        fileMemory.remove(rmkey);
                        RedisUtils.delCacheMapValue(fileMemoryKey, rmkey);
                    }
            );

        }
    }

    public boolean stopListener() {
        this.stop = Boolean.TRUE;
        this.fileMemory.clear();
        return this.stop;
    }
}