# -*- coding:utf-8 -*-
import ftplib
import functools
import os
import json
import time
import traceback
import schedule
from ftplib import FTP
import logging
from setting import Config
from logging.handlers import TimedRotatingFileHandler
logger = logging.getLogger("loggername")
# 设置logger的level为DEBUG
logger.setLevel(logging.DEBUG)
# 创建一个输出日志到控制台的StreamHandler
stream_handler = logging.StreamHandler()
formatter = logging.Formatter('[%(asctime)s]:[%(filename)s]:%(name)s:%(levelname)s: %(message)s')
stream_handler.setFormatter(formatter)
log_path = "./log"
os.makedirs(log_path, exist_ok=True)
# 把log输出到当前目录下交usk.log的文件
filename = '{}/log.log'.format(log_path)
file_handler = TimedRotatingFileHandler(filename, when='midnight', interval=1, backupCount=10)
file_handler.setFormatter(formatter)
logger.addHandler(stream_handler) # 把日志打印到控制台
logger.addHandler(file_handler) # 把日志打印到文件
class FTPClient():
"""
"""
def __init__(self, host, port, username, password):
self.host = host
self.port = port
self.username = username
self.password = password
self.ftp = None
self.bufsize = 1024
self.make_conn()
self.workpath = None
def make_conn(self):
"""
创建 FTP 实例
"""
if not self.ftp:
ftp = FTP()
ftp.connect(self.host, self.port)
ftp.login(user=self.username, passwd=self.password)
ftp.set_debuglevel(3)
self.ftp = ftp
return self.ftp
def is_valid_path(self, path):
"""
判断输入路径合法性
"""
dirname = os.path.dirname(path)
basename = os.path.basename(path)
return bool(dirname) and bool(basename)
def get_all_files(self, path: str, ext: list = None):
"""
获取文件夹下所有文件绝对路径
:param path:
:param ext: 后缀列表[".txt",".json",...]
:return:
"""
try:
if os.path.exists(path) and os.path.isabs(path):
for path, dir_lis, file_lis in os.walk(path):
if len(file_lis) > 0:
for name in file_lis:
if ext:
if os.path.splitext(name)[-1] in ext:
yield {
"name": name,
"file": os.path.join(path, name),
}
else:
yield {
"name": name,
"file": os.path.join(path, name),
}
except BaseException:
print(traceback.format_exc())
def upload(self, local_file, remotefile):
"""
"""
print("upload---local_file, remotefile", local_file, remotefile)
self.make_conn()
if not self.workpath:
self.workpath = self.ftp.pwd()
print("self.workpath", self.workpath)
fp = open(local_file, 'rb')
split_remotefile = remotefile.split(os.sep)
print("split_remotefile", split_remotefile)
print("split_remotefile[:-1]", split_remotefile[:-1])
# 建新文件夹
if self.is_valid_path(remotefile):
for d in split_remotefile[:-1]:
if not d:
continue
try:
self.ftp.cwd(d)
except ftplib.error_perm:
self.ftp.mkd(d)
self.ftp.cwd(d)
self.ftp.pwd()
self.ftp.cwd(self.workpath)
# self.ftp.storbinary('STOR ' + split_remotefile[-1], fp, self.bufsize)
fp.close()
def main(self, origin_path, remote_path):
print("origin_path, remote_path", origin_path, remote_path)
f = open("config.json", "r", encoding="utf-8")
start_time = json.loads(f.read())["start"]
print("start_time", start_time)
f.close()
year = int(start_time[:2])
month = int(start_time[2:4])
day = int(start_time[-2:])
print("year,month,day", year, month, day)
if day >= 31:
if month >= 12:
year += 1
month = 1
day = 1
else:
month += 1
day = 1
else:
day += 1
_new_time = "{}{}{}".format(
str(year).zfill(2),
str(month).zfill(2),
str(day).zfill(2))
print("_new_time", _new_time)
new_time = "{}{}\\{}".format(
str(year).zfill(2),
str(month).zfill(2),
str(day).zfill(2))
print("new_time", new_time)
new_local_file = origin_path + "\\\\" + new_time
new_local_file = new_local_file.replace("\\\\", "\\")
print("new_local_file", new_local_file)
if os.path.exists(new_local_file):
if len(os.listdir(new_local_file)) <= 0:
logger.error("空 {}".format(new_local_file))
else:
logger.info("开始处理:{}".format(new_time))
try:
fclient = FTPClient(
self.host, self.port, self.username, self.password)
for local_file in fclient.get_all_files(new_local_file):
print("local_file", local_file)
file_tail = local_file["file"].replace(origin_path, "")
print("file_tail", file_tail)
remote_file = remote_path + file_tail
print("remote_file", remote_file)
try:
fclient.upload(
local_file["file"].replace(
"\\\\", "\\"), remote_file)
except BaseException:
logger.exception(
"文件上传失败:{}".format(
local_file["file"].replace(
"\\\\", "\\")))
logger.info("结束处理:{}".format(new_time))
except BaseException:
logger.exception(
"连接失败:{}{}{}{}".format(
host, port, username, password))
else:
logger.error("不存在:{}".format(new_local_file))
# with open("config.json", "w", encoding="utf-8")as fp:
# fp.write(json.dumps({
# "start": _new_time
# }))
def job(origin_path, remote_path, fc):
fc.main(origin_path, remote_path)
if __name__ == '__main__':
host, port, username, password = Config.get("host"), Config.get(
"port"), Config.get("username"), Config.get("password")
fc = FTPClient(host, port, username, password)
fc.main(Config.get("origin_path"), Config.get("remote_path"))
# 10s调用一次
# schedule.every(10).seconds.do(
# functools.partial(
# job,
# Config.get("origin_path"),
# Config.get("remote_path"),
# fc))
# 每小时
# schedule.every(1).hours.do(functools.partial(
# job,
# Config.get("origin_path"),
# Config.get("remote_path"),
# fc))
# 每天9点
# schedule.every().day.at("09:00").do(functools.partial(
# Config.get("origin_path"),
# Config.get("remote_path"),
# fc))
# while True:
# schedule.run_pending()
# time.sleep(1)
