springboot 集合springcamel 读取ftp文件,入mongdb

前言

 最近与第三方公司联调,最终决定第三方把数据存入ftp服务器一个是csv文件,80w条数据,一个是txt文件,每天三万

选择技术以后决定使用camel 读取ftp文件入库!

    首先引入camel jar包

 
            org.apache.camel
            camel-spring-boot-starter
            2.19.1
        
        
            org.apache.camel
            camel-bindy
            2.19.1
        
        
            org.apache.camel
            camel-ftp-starter
            2.19.1

        

在yml文件写入ftp的配置

ftp:
  servername: ftp服务器IP
  port: 端口
  user: 账号
  password: 密码
  directory: 读取文件夹
  file: sample.txt
  local.dir: file:target/download
  errorTileUrl: file:target/download/error?noop=true
  delay: 10s//设置多久读取一次ftp文件夹

  systemCharSet: utf-8

package com.tianyi.joe.ftpProcessor;

//import javax.validation.ValidationException;

import org.apache.camel.builder.RouteBuilder;
//import org.apache.camel.dataformat.bindy.csv.BindyCsvDataFormat;
//import org.apache.camel.model.dataformat.BindyDataFormat;
import org.apache.camel.model.dataformat.BindyType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;
import com.tianyi.joe.domain.DeviceData;
/**
 * Created by Asa
 */
@Component
@Lazy(true)
public class CustomRouteBuilder extends RouteBuilder {

	private static final Logger LOGGER = LoggerFactory
			.getLogger(CustomRouteBuilder.class);
	@Autowired
	private AlarmDeviceDataProcessor alarmDeviceDataProcessor;

	
	@Value("${ftp.servername}")
	private String serverName;

	@Value("${ftp.user}")
	private String user;

	@Value("${ftp.port}")
	private String port;

	@Value("${ftp.password}")
	private String password;

	@Value("${ftp.directory}")
	private String directory;
	
	@Value("${param.size}")
	private int size;
	@Value("${ftp.delay}")
	private String delay;
	@Value("${ftp.systemCharSet}")
	private String systemCharSet;
	
	@Override
	public void configure() throws Exception {
		LOGGER.info("Initializing routes ...");
		String uri = String
				.format("ftp://%s@%s:%s/%s?password=%s&stepwise=true&passiveMode=true&delete=false&filter=#textFileFilter&delay=%s&readLock=rename&ftpClient.controlEncoding=%s&move=./back/${date:now:yyyyMMddHHmmss}-${file:name}",
						user, serverName, port, directory, password, delay,
						systemCharSet);
		
		String errorTileUrl = "file:target/download/error?noop=true";
		from(uri).onException(IllegalArgumentException.class).handled(true)
				.to(errorTileUrl).end().split().tokenize("\n", 10000)
				.threads(10, 20, "测试").unmarshal()
				.bindy(BindyType.Csv, DeviceData.class)
				.process(alarmDeviceDataProcessor)
				.log("Downloaded file ${file:name} complete.");
		
	}
}

读取ftp文件,并且把ftp文件移入到.back目录下

package com.tianyi.joe.ftpProcessor.filters;

import org.apache.camel.component.file.GenericFile;
import org.apache.camel.component.file.GenericFileFilter;
import org.springframework.stereotype.Component;

/**
 *
 */
@Component("textFileFilter")
public class TextFileFilter implements GenericFileFilter {
	@Override
	public boolean accept(GenericFile file) {
		return file.getFileName().endsWith(".csv")||file.getFileName().endsWith(".txt");
	}
}

读取ftp文件的格式  (为csv和txt)

package com.tianyi.joe.ftpProcessor;

import java.util.Collection;

import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;

import com.tianyi.joe.domain.DeviceData;
import com.tianyi.joe.service.GzAlarmDeviceYwStateService;

/**
 * 
 * 
 * 
 */
@Component
public class AlarmDeviceDataProcessor implements Processor {

	final Logger logger = LoggerFactory.getLogger(AlarmDeviceDataProcessor.class);
    @Autowired
    Environment environment;
    
    @Autowired
    private  GzAlarmDeviceYwStateService gzAlarmDeviceYwStateService;
    
   
    
    @SuppressWarnings("unchecked")
   	private void processMarks(Exchange ex) {
 
   		if (ex.getIn().getBody().getClass().isAssignableFrom(DeviceData.class)) {
   			gzAlarmDeviceYwStateService
			   .saveAlarmDeviceServicestates((DeviceData) ex.getIn().getBody());
		} else {
			gzAlarmDeviceYwStateService.saveAlarmDeviceServicestates((Collection) ex.getIn().getBody());
		}
   	}
  
	@Override
	public void process(final Exchange exchange) throws Exception {
		processMarks(exchange);
	}

}

实现 Processor  剩下的就是一些自己需要的逻辑了