Apache Camel FTP + Spring Boot 配置简述

说明

由于之前项目中已经使用 apache camel ftp 工具来处理 ftp 相关操作，这里沿用该工具。简要配置如下：

工具包

<dependency>
    <groupId>org.apache.camelgroupId>
    <artifactId>camel-spring-boot-starterartifactId>
    <version>2.19.4version>
dependency>

<dependency>
    <groupId>org.apache.camelgroupId>
    <artifactId>camel-ftpartifactId>
    <version>2.19.4version>
dependency>

配置文件示例

说明：

• localWorkDirectory : 本地工作目录，由于本项目使用的 ftp 接口文件较大，为了防止其直接加载到内存导致内存溢出，配置本地工作目录来缓存下载中的文件。
• useUserKnownHostsFile : 是否使用主机本地的known_hosts，配置为false，则会默认到classpath下查找。
• filter : 过滤方法。

ftp.server.uri=sftp://${ftp.url}\
               ?username=${ftp.username}\
               &password=${ftp.password}\
               &useUserKnownHostsFile=false\
               &localWorkDirectory=${ftp.local.work.directory}\
               &delay=1h\
               &readLock=rename\
               &filter=#ftpDownloadFileFilter\
               &stepwise=false

# 用户名密码
ftp.url=192.168.192.128:22022
ftp.username=ftp_test
ftp.password=ftp_test_pw

# 本地工作目录
ftp.local.work.directory=D:/TestData/sunknew/sftp/temp
# 本地同步目录
ftp.local.data.dir=D:/TestData/sunknew/sftp/data

known_hosts示例

[192.168.192.128]:22022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIpYfg3lLmpP/NKEi5aF9ReY42JOBYf40gkdcIZe7hi+KXgvYOwSjSnZ1O71gaeJvTPk8UU01j8tgQegCNwSNqo=
[192.168.192.128]:22022 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxf7XvwmHHjtI3wJwURMd4hV+pP6md7+g+FouZNWecHarhen+bS795EFGvYa4c/srKRg4/Kj70gF0wV62gDkXGgrNMR5mFr8NGR0fFoA5E0K2k7bZNvm2+/ngU6Sno/olbb3IlfQ2c2YmnFqaDPIINOnyZC4UyJA3Wjm1xGf7zrUn3PQMpS4+QmvhpNj2JDzQfR8cb429Cjmw0JUrtJRrfPNlQUaqeXVl6RmyZw+Np/yXf/4KSYs4SN1P9+0LF00Qewm6ch70N8vnY3DyRRDx24oQohU72gjkJsA8bEkLxZHEzg52qpaZaN/+sl2NrmebarCXl+/eOkSqqm+Ejcv5rw==

Filter示例

package com.sunknew.ftp.filter;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Calendar;
import java.util.List;
import java.util.stream.Collectors;

import org.apache.camel.component.file.GenericFile;
import org.apache.camel.component.file.GenericFileFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

@Component
public class FtpDownloadFileFilter implements GenericFileFilter<Object> {

    private static Logger logger = LoggerFactory.getLogger(FtpDownloadFileFilter.class);

    @Value("${ftp.local.data.dir}")
    private String localDir;

    /**
     * 过滤下载文件
     * @author sunk
     */
    @Override
    public boolean accept(GenericFile<Object> file) {

        try {
            String fileName = file.getFileName();
            long lastModified = file.getLastModified();

            return isLatestFile(lastModified, 3) && !isInLocalDir(fileName) ? true : false;
        } catch (Exception e) {
            logger.error("ftp download file filter error !", e);
            return false;
        }
    }

    /**
     * 文件是否为最近几天内的
     * 
 用于过滤一部分历史数据
     * @author sunk
     */
    private boolean isLatestFile(long lastModified, int dateNum) {

        Calendar calLastMod = Calendar.getInstance();
        calLastMod.setTimeInMillis(lastModified);

        Calendar calThreshold = Calendar.getInstance();
        calThreshold.add(Calendar.DATE, - dateNum);

        return calLastMod.compareTo(calThreshold) > 0 ? true : false;
    }

    /**
     * 文件是否已在本地目录中
     * @author sunk
     */
    private boolean isInLocalDir(String fileName) {
        try {
            //获取本地文件夹中已下载的文件名
            List<String> localFileNames = Files.walk(Paths.get(localDir))
                    .filter(Files::isRegularFile)
                    .map(Path::getFileName)
                    .map(Path::toString)
                    .collect(Collectors.toList());

            return localFileNames.contains(fileName) ? true : false;
        } catch (Exception e) {
            logger.error("get local downloaded files fail !", e);
            return true;
        }
    }
}

Route示例

由于本项目中文件较大，不在此处配置处理方法

package com.sunknew.ftp.task;

import java.net.InetAddress;

import org.apache.camel.LoggingLevel;
import org.apache.camel.builder.RouteBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

@Component
public class FtpDownloadRoute extends RouteBuilder {

    private static Logger logger = LoggerFactory.getLogger(FtpDownloadRoute.class);

    @Value("${ftp.server.uri}")
    private String ftpUri;

    @Value("${ftp.local.data.dir}")
    private String localDir;

    @Value("${host.nodes}")
    private String hostNodes;

    @Value("${ftp.task.start}")
    private boolean isStart;

    @Override
    public void configure() throws Exception {

        if (isStart && isExecHost()) {
            from(ftpUri).to("file:" + localDir)
                .log(LoggingLevel.INFO, logger, "download file ${file:name} complete.");
        }
    }

    /**
     * 判断是否为执行任务的主机
     * @author sunk
     */
    private boolean isExecHost() {
        String hostName = "";
        try {
            hostName = InetAddress.getLocalHost().getHostName();
        } catch (Exception e) {
            logger.error("get hostname fail !", e);
            return false;
        }
        return hostName.endsWith(hostNodes.split(",")[0]);
    }
}

参考

1.Apache Camel: FTP2
2.Apache Camel: File2
3.Apache Camel: Spring Boot
4.Apache Camel 与 Spring Boot 集成，通过FTP定时采集、处理文件