Spring Boot实现SFTP文件上传下载
Spring Boot实现SFTP文件上传下载
1.实现背景及现实意义
近期由于系统迁移到docker容器，采用Spring Boot 框架实现微服务治理，在此基础上晚间批量文件服务器也由ftp改成sftp，由于之前ftp的实现是采用公具类的形式，在此基础之上，未屏蔽开发细节和依赖Spring Boot自动装配的特性，进行组件的自动装配和改造，旨在实现简化开发，提高文件传输的安全性和数据交互的可靠性。
2.什么是sftp
sftp是SSH File Transfer Protocol的缩写，安全文件传送协议。可以为传输文件提供一种安全的网络的加密方法。sftp 与 ftp 有着几乎一样的语法和功能。SFTP 为 SSH的其中一部分，是一种传输档案至 Blogger 伺服器的安全方式。其实在SSH软件包中，已经包含了一个叫作SFTP(Secure File Transfer Protocol)的安全文件信息传输子系统，SFTP本身没有单独的守护进程，它必须使用sshd守护进程（端口号默认是22）来完成相应的连接和答复操作，所以从某种意义上来说，SFTP并不像一个服务器程序，而更像是一个客户端程序。SFTP同样是使用加密传输认证信息和传输的数据，所以，使用SFTP是非常安全的。但是，由于这种传输方式使用了加密/解密技术，所以传输效率比普通的FTP要低得多。
3.sftp文件传输在java中的实现一
3.1Maven依赖
<dependency>
<groupId>com.jcraftgroupId>
<artifactId>jschartifactId>
<version>0.1.54version>
dependency>
3.2sftp相关config封装
/**
* @ClassName: SftpConfig
* @Description: sftp配置类
* @Author: 尚先生
* @CreateDate: 2019/1/7
* @Version: 1.0
*/
public class SftpConfig {
private String hostname;
private Integer port;
private String username;
private String password;
private Integer timeout;
private Resource privateKey;
private String remoteRootPath;
private String fileSuffix;
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public Integer getTimeout() {
return timeout;
}
public void setTimeout(Integer timeout) {
this.timeout = timeout;
}
public Resource getPrivateKey() {
return privateKey;
}
public void setPrivateKey(Resource privateKey) {
this.privateKey = privateKey;
}
public String getRemoteRootPath() {
return remoteRootPath;
}
public void setRemoteRootPath(String remoteRootPath) {
this.remoteRootPath = remoteRootPath;
}
public String getFileSuffix() {
return fileSuffix;
}
public void setFileSuffix(String fileSuffix) {
this.fileSuffix = fileSuffix;
}
public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, Resource privateKey, String remoteRootPath, String fileSuffix) {
this.hostname = hostname;
this.port = port;
this.username = username;
this.password = password;
this.timeout = timeout;
this.privateKey = privateKey;
this.remoteRootPath = remoteRootPath;
this.fileSuffix = fileSuffix;
}
public SftpConfig(String hostname, Integer port, String username, String password, Integer timeout, String remoteRootPath) {
this.hostname = hostname;
this.port = port;
this.username = username;
this.password = password;
this.timeout = timeout;
this.remoteRootPath = remoteRootPath;
}
public SftpConfig() {
}
}
3.3sftp工具类实现
/**
* @ClassName: SFTP
* @Description: sftp上传通用类
* @Author: 尚先生
* @CreateDate: 2019/1/3
* @Version: 1.0
*/
public class SFTP {
private long count;
/**
* 已经连接次数
*/
private long count1 = 0;
private long sleepTime;
private static final Logger logger = LoggerFactory.getLogger(SFTP.class);
/**
* 连接sftp服务器
*
* @return
*/
public ChannelSftp connect(SftpConfig sftpConfig) {
ChannelSftp sftp = null;
try {
JSch jsch = new JSch();
jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort());
Session sshSession = jsch.getSession(sftpConfig.getUsername(), sftpConfig.getHostname(), sftpConfig.getPort());
logger.info("Session created ... UserName=" + sftpConfig.getUsername() + ";host=" + sftpConfig.getHostname() + ";port=" + sftpConfig.getPort());
sshSession.setPassword(sftpConfig.getPassword());
Properties sshConfig = new Properties();
sshConfig.put("StrictHostKeyChecking", "no");
sshSession.setConfig(sshConfig);
sshSession.connect();
logger.info("Session connected ...");
logger.info("Opening Channel ...");
Channel channel = sshSession.openChannel("sftp");
channel.connect();
sftp = (ChannelSftp) channel;
logger.info("登录成功");
} catch (Exception e) {
try {
count1 += 1;
if (count == count1) {
throw new RuntimeException(e);
}
Thread.sleep(sleepTime);
logger.info("重新连接....");
connect(sftpConfig);
} catch (InterruptedException e1) {
throw new RuntimeException(e1);
}
}
return sftp;
}
/**
* 上传文件
*
* @param directory 上传的目录
* @param uploadFile 要上传的文件
* @param sftpConfig
*/
public void upload(String directory, String uploadFile, SftpConfig sftpConfig) {
ChannelSftp sftp = connect(sftpConfig);
try {
sftp.cd(directory);
} catch (SftpException e) {
try {
sftp.mkdir(directory);
sftp.cd(directory);
} catch (SftpException e1) {
throw new RuntimeException("ftp创建文件路径失败" + directory);
}
}
File file = new File(uploadFile);
InputStream inputStream=null;
try {
inputStream = new FileInputStream(file);
sftp.put(inputStream, file.getName());
} catch (Exception e) {
throw new RuntimeException("sftp异常" + e);
} finally {
disConnect(sftp);
closeStream(inputStream,null);
}
}
/**
* 下载文件
*
* @param directory 下载目录
* @param downloadFile 下载的文件
* @param saveFile 存在本地的路径
* @param sftpConfig
*/
public void download(String directory, String downloadFile, String saveFile, SftpConfig sftpConfig) {
OutputStream output = null;
try {
File localDirFile = new File(saveFile);
// 判断本地目录是否存在，不存在需要新建各级目录
if (!localDirFile.exists()) {
localDirFile.mkdirs();
}
if (logger.isInfoEnabled()) {
logger.info("开始获取远程文件:[{}]---->[{}]", new Object[]{directory, saveFile});
}
ChannelSftp sftp = connect(sftpConfig);
sftp.cd(directory);
if (logger.isInfoEnabled()) {
logger.info("打开远程文件:[{}]", new Object[]{directory});
}
output = new FileOutputStream(new File(saveFile.concat(File.separator).concat(downloadFile)));
sftp.get(downloadFile, output);
if (logger.isInfoEnabled()) {
logger.info("文件下载成功");
}
disConnect(sftp);
} catch (Exception e) {
if (logger.isInfoEnabled()) {
logger.info("文件下载出现异常，[{}]", e);
}
throw new RuntimeException("文件下载出现异常，[{}]", e);
} finally {
closeStream(null,output);
}
}
/**
* 下载远程文件夹下的所有文件
*
* @param remoteFilePath
* @param localDirPath
* @throws Exception
*/
public void getFileDir(String remoteFilePath, String localDirPath, SftpConfig sftpConfig) throws Exception {
File localDirFile = new File(localDirPath);
// 判断本地目录是否存在，不存在需要新建各级目录
if (!localDirFile.exists()) {
localDirFile.mkdirs();
}
if (logger.isInfoEnabled()) {
logger.info("sftp文件服务器文件夹[{}],下载到本地目录[{}]", new Object[]{remoteFilePath, localDirFile});
}
ChannelSftp channelSftp = connect(sftpConfig);
Vector<LsEntry> lsEntries = channelSftp.ls(remoteFilePath);
if (logger.isInfoEnabled()) {
logger.info("远程目录下的文件为[{}]", lsEntries);
}
for (LsEntry entry : lsEntries) {
String fileName = entry.getFilename();
if (checkFileName(fileName)) {
continue;
}
String remoteFileName = getRemoteFilePath(remoteFilePath, fileName);
channelSftp.get(remoteFileName, localDirPath);
}
disConnect(channelSftp);
}
/**
* 关闭流
* @param outputStream
*/
private void closeStream(InputStream inputStream,OutputStream outputStream) {
if (outputStream != null) {
try {
outputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(inputStream != null){
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private boolean checkFileName(String fileName) {
if (".".equals(fileName) || "..".equals(fileName)) {
return true;
}
return false;
}
private String getRemoteFilePath(String remoteFilePath, String fileName) {
if (remoteFilePath.endsWith("/")) {
return remoteFilePath.concat(fileName);
} else {
return remoteFilePath.concat("/").concat(fileName);
}
}
/**
* 删除文件
*
* @param directory 要删除文件所在目录
* @param deleteFile 要删除的文件
* @param sftp
*/
public void delete(String directory, String deleteFile, ChannelSftp sftp) {
try {
sftp.cd(directory);
sftp.rm(deleteFile);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 列出目录下的文件
*
* @param directory 要列出的目录
* @param sftpConfig
* @return
* @throws SftpException
*/
public List<String> listFiles(String directory, SftpConfig sftpConfig) throws SftpException {
ChannelSftp sftp = connect(sftpConfig);
List fileNameList = new ArrayList();
try {
sftp.cd(directory);
} catch (SftpException e) {
return fileNameList;
}
Vector vector = sftp.ls(directory);
for (int i = 0; i < vector.size(); i++) {
if (vector.get(i) instanceof LsEntry) {
LsEntry lsEntry = (LsEntry) vector.get(i);
String fileName = lsEntry.getFilename();
if (".".equals(fileName) || "..".equals(fileName)) {
continue;
}
fileNameList.add(fileName);
}
}
disConnect(sftp);
return fileNameList;
}
/**
* 断掉连接
*/
public void disConnect(ChannelSftp sftp) {
try {
sftp.disconnect();
sftp.getSession().disconnect();
} catch (Exception e) {
e.printStackTrace();
}
}
public SFTP(long count, long sleepTime) {
this.count = count;
this.sleepTime = sleepTime;
}
public SFTP() {
}
}
3.4测试类实现
/**
* @ClassName: TestSFTPUtils
* @Description: SFTP工具类测试类
* @Author: 尚先生
* @CreateDate: 2019/4/25 11:09
* @Version: 1.0
*/
public class TestSFTPUtils {
private static final Logger logger = LoggerFactory.getLogger(TestSFTPUtils.class);
public static void main(String[] args) {
SFTP ftp = new SFTP(3, 6000);
SftpConfig sftpConfig = new SftpConfig("10.0.155.55", 22, "test", "test", 1000, "/opt/bdepfile/bdp/tset/20190425");
try {
List<String> list = ftp.listFiles("/opt/bdepfile/bdp/pucms/20190108", sftpConfig);
logger.info("文件上传下载详情" ,new Object[]{list});
} catch (SftpException e) {
logger.error("文件上传下载异常:[{}]" ,new Object[]{e});
}
}
}
4.sftp文件传输在java中的实现二
4.1Maven依赖
<dependency>
<groupId>com.jcraftgroupId>
<artifactId>jschartifactId>
<version>0.1.54version>
dependency>
4.2扩展上述工具类的实现
4.2.1SFTP.java改造
/**
* @ClassName: SFTP
* @Description: sftp上传通用类
* @Author: 尚先生
* @CreateDate: 2019/1/3
* @Version: 1.0
*/
@Component
public class SFTP {
private ChannelSftp channelSftp;
@Value("${sftp.remotepath}")
private String remotepath;
@Value("${sftp.localpath}")
private String localpath;
@Value("${sftp.filenames}")
private String filenames;
private static final String COMPLATEG_FILE_FLAG = "over_%s.dat";
private static final Logger logger = LoggerFactory.getLogger(SFTP.class);
...
public void setChannelSftp(ChannelSftp channelSftp) {
this.channelSftp = channelSftp;
}
/**
*
* @param remotePath
* @param remoteFileName
* @param localPath
* @param localFileName
* @return
*/
public boolean downloadFile(String remotePath, String remoteFileName, String localPath, String localFileName){
logger.info("开始下载文件，远程路径:[{}],本地路径:[{}],文件名称:[{}]",new Object[]{remotePath,localPath,remoteFileName});
FileOutputStream fileOutputStream = null;
File file = new File(localPath + localFileName);
try {
fileOutputStream = new FileOutputStream(file);
channelSftp.get(remotePath + remoteFileName,fileOutputStream);
return true;
} catch (Exception e) {
logger.error("sftp下载文件失败:[{}]",new Object[]{e});
return false;
}
}
/**
* 单个ok文件下载
* @param trandate
* @return
*/
public boolean downloadOKFile(String trandate){
trandate = trandate.replace("-", "");
String localDirPath = localpath.concat("/").concat(trandate);
File localDirFile = new File(localDirPath);
if (!localDirFile.exists()){
localDirFile.mkdirs();
}else {
logger.info("文件[{}]已存在",new Object[]{localDirPath});
if (!localDirFile.isDirectory()){
logger.error("文件[{}]已存在，但不是目录，文件下载失败",new Object[]{localDirPath});
throw new RuntimeException(String.format("本地文件[{%s}]已存在，但不是目录，不能创建文件",localDirPath));
}
}
String filename = String.format(COMPLATEG_FILE_FLAG, trandate);
String remoteFilePath = remotepath.concat("/").concat(trandate).concat("/");
String localFilePath = localDirPath.concat("/");
boolean flag = downloadFile(remoteFilePath, filename, localFilePath, filename);
return flag;
}
/**
* 多个文件下载
* @param trandate
* @return
*/
public boolean downloadCoreFilesToLocal(String trandate){
boolean flag = false;
trandate = trandate.replace("-", "");
String localDirPath = localpath.concat("/").concat(trandate).concat("/");
String remoteDirPath = remotepath.concat("/").concat(trandate).concat("/");
for (String coreFileName : filenames.split(",")){
//文件名称截取
String coreFilaName = String.format(coreFileName.trim(), trandate);
flag = downloadFile(remoteDirPath, coreFileName, localDirPath, coreFileName);
}
return flag;
}
}
4.2.2自动装配Sftp连接器
/**
* @ClassName: SftpClientConfigure
* @Description: 自动装配Sftp连接器
* @Author: 尚先生
* @CreateDate: 2019/4/25
* @Version: 1.0
*/
@Configuration
@ConfigurationProperties(prefix = "sftp")
public class SftpClientConfigure {
private String hostname;
private Integer port;
private String username;
private String password;
private Integer timeout;
private String privateKey;
private String remoteRootPath;
private String fileSuffix;
// 通道类型
private static final String CHANNEL_TYPE = "sftp";
private static final Logger logger = LoggerFactory.getLogger(SftpClientConfigure.class);
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getPrivateKey() {
return privateKey;
}
public void setPrivateKey(String privateKey) {
this.privateKey = privateKey;
}
public String getRemoteRootPath() {
return remoteRootPath;
}
public void setRemoteRootPath(String remoteRootPath) {
this.remoteRootPath = remoteRootPath;
}
public String getFileSuffix() {
return fileSuffix;
}
public void setFileSuffix(String fileSuffix) {
this.fileSuffix = fileSuffix;
}
public Integer getTimeout() {
return timeout;
}
public void setTimeout(Integer timeout) {
this.timeout = timeout;
}
@Bean("sshSession")
@Lazy
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public Session session() throws JSchException {
if (logger.isInfoEnabled()) {
logger.info("获取session，设置的超时时间为[{}]毫秒", timeout);
}
JSch jsch = new JSch();
Session session = jsch.getSession(username, hostname, port);
// 设置秘钥
// jsch.addIdentity(privateKey);
session.setPassword(password);
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config); //为Session对象设置properties
session.setTimeout(timeout); //设置timeout时间
session.connect(); //通过Session建立链接
return session;
}
@Bean("coreSftpChannel")
@Lazy
@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE)
public ChannelSftp channel(Session session) throws JSchException {
if (logger.isInfoEnabled()) {
logger.info("初始化sftp连接");
}
Channel channel = session.openChannel(CHANNEL_TYPE); //打开SFTP通道
channel.connect(); //建立SFTP通道的连接
return (ChannelSftp) channel;
}
}
4.2.3配置文件
# sftp配置
sftp.hostname=10.0.155.55
sftp.port=22
sftp.username=test
sftp.password=test
sftp.timeout=6000
sftp.privateKey=
sftp.remotepath=/opt/bdepfile/bdp/tset
sftp.localpath=D:/core
sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat
4.2.4测试类实现
/**
* @ClassName: TestAutoConfigurationSFTP
* @Description: SftpClientConfigure测试类
* @Author: 尚先生
* @CreateDate: 2019/4/25
* @Version: 1.0
*/
@RunWith(SpringRunner.class)
@SpringBootTest
public class TestAutoConfigurationSFTP {
private static final Logger logger = LoggerFactory.getLogger(TestAutoConfigurationSFTP.class);
@Autowired
@Qualifier("coreSftpChannel")
private ChannelSftp channelSftp;
@Autowired
private SFTP sftp;
@Test
public void testAotuDownload(){
String trandate = "2019-04-25";
boolean flag = false;
sftp.setChannelSftp(channelSftp);
flag = sftp.downloadOKFile(trandate);
flag = sftp.downloadCoreFilesToLocal(trandate);
logger.error("下载文件结果：[{}]",new Object[]{flag});
}
}
4.2.5测试结果
打开 Git Bash Here
cd D:\core
ll
20190425
cd 20190425
ll -als
./
../
duebillInfo_20190425.dat
repayInfo_20190425.dat
over_20190425.dat
4.3sftp扩展实现
由于当前Spring Boot环境实现，采用的是Spring Boot"自动装配"实现的，自动注入并调用实现从核心拉取文件的功能。
-
在项目中取固定文件时，只需动态追加或者替换下面的配置项
sftp.filenames=duebillInfo_%s.dat,repayInfo_%s.dat -
如果是新增sftp连接时可以手动创建
SftpConfig config = new SftpConfig();SFTP sftp = new SFTP();sftp.download(directory, downloadFile, saveFile, sftpConfig) -
至此，多种方式实现sftp文件传输可以共存，而且在系统中可以实现"自动装配"。
完整代码和相关依赖请见GitHub
https://github.com/dwyanewede/project-learn/tree/master/src/main/java/com/learn/demo/sftp
相关文章推荐
Spring Framework集成FTP文件上传下载
https://blog.csdn.net/shang_xs/article/details/89510574
公众号推荐
