python_定时任务自动匹配数据

从配置文件读取配置参数，并每天定时执行日常任务、每周一定时执行周任务，匹配符合要求的pos数据。

import os
import datetime
import re
import shutil
import time
import threading
import schedule
from dateutil import parser

# 全局变量：配置文件
glob_config = "configss.txt"


def txt_to_dict(filename):
    with open(filename, 'r', encoding='utf-8') as f:
        data_dict = {}
        for line in f:
            # 忽略空行和注释行
            if line.strip() == '' or line.startswith('#'):
                continue
            key, value = line.strip().split('=')
            data_dict[key] = value
    return data_dict


def copy_exchange(src_path, dst_path, pattern_start, pattern_end):

    # 示例：将2023年10月19日12时（UTC时间）转换为GPS周和周内秒
    utc_time_s = datetime.datetime.strptime(pattern_start, '%Y-%m-%d %H:%M:%S')
    utc_time_e = datetime.datetime.strptime(pattern_end, '%Y-%m-%d %H:%M:%S')
    # GPS周起始时间（1980年1月6日）
    gps_epoch = datetime.datetime(1980, 1, 6)

    # 计算时间差
    time_difference_s = utc_time_s - gps_epoch
    time_difference_e = utc_time_e - gps_epoch

    # 计算总秒数和周数，gps时间比utc时间快了18秒
    total_seconds_s = time_difference_s.total_seconds() + 18
    gps_week_s = total_seconds_s // (7 * 24 * 3600)
    gps_week_seconds_s = total_seconds_s % (7 * 24 * 3600)
    pattern_start_ss = str(int(gps_week_s)) + " " + str(gps_week_seconds_s)
    total_seconds_e = time_difference_e.total_seconds() + 18
    gps_week_e = total_seconds_e // (7 * 24 * 3600)
    gps_week_seconds_e = total_seconds_e % (7 * 24 * 3600)
    pattern_end_ee = str(int(gps_week_e)) + " " + str(gps_week_seconds_e)

    # 文件命名
    now = datetime.datetime.now()
    p = pattern_start.split()
    fn = re.sub(r'-', '', p[0])
    fnn = re.sub(r':', '', p[1])
    # 区分不同环境的pos文件
    if "PROD_V2_N" in src_path:
        print("PROD_V2_N环境的SSR文件")
        file_name = "PROD_N_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
    elif "PROD_V2" in src_path:
        print("PROD_V2环境的SSR文件")
        file_name = "PROD_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
    elif "TEST_V2_P" in src_path:
        print("TEST_V2_P环境的SSR文件")
        file_name = "TEST_P_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
    elif "TEST_V2" in src_path:
        print("TEST_V2环境的SSR文件")
        file_name = "TEST_" + fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'
    else:
        print("非PROD和TEST环境的SSR文件")
        file_name = fn + '_' + fnn + now.strftime('_%H%M%S') + '.pos'

    # 复制文件
    dst_path_c = os.path.join(dst_path, file_name)
    shutil.copy2(src_path, dst_path_c)

    dst_w_path = os.path.join(dst_path, 'N_' + file_name)
    # 写数据
    with open(dst_path_c, 'r') as dst, open(dst_w_path, 'w') as dst_w:
        # 先写入文件前两行
        lineTwo = dst.readlines()[:2]
        dst_w.writelines(lineTwo)
    with open(dst_path_c, 'r') as dst, open(dst_w_path, 'a+') as dst_w:
        # 再匹配
        # 默认不写入文件
        writing = False
        num_count = 0
        for line in dst:
            # 正则匹配，起始行
            if re.search(pattern_start_ss, line):
                writing = True
            if writing:
                num_count += 1
                dst_w.write(line)
            # 正则匹配，截止行
            if re.search(pattern_end_ee, line):
                writing = False
                break
        if num_count == 0:
            print(r'当前时间：%s，没有数据匹配' % (now.strftime('%Y-%m-%d %H:%M:%S')))
        else:
            print(r'当前时间：%s，总共%s行写入完成，起始于:%s，截止于:%s' % (
                now.strftime('%Y-%m-%d %H:%M:%S'), num_count, pattern_start, pattern_end))

    if dst_path_c:
        os.remove(dst_path_c)


def check_date_format(date_string):
    # try:
    #     parser.parse(date_string)
    #     return True
    # except ValueError:
    #     return False

    # 判断是否符合年月日格式，格式为YYYY-MM-DD
    date_pattern = r'^\d{4}-\d{2}-\d{2}$'
    match_date = date_string.split()
    # print(match_date[0])
    if re.match(date_pattern, match_date[0]):
        return True
    else:
        return False


# 日常任务
def job_test():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_test = data.get('origin_path_test', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start = data.get('date_start', 'null')
    date_end = data.get('date_end', 'null')

    # 根据路径决定是否执行任务
    if origin_path_test:
        # 判断时间日期格式
        if check_date_format(date_start):
            print(f'{date_start}:年月日时分秒格式')
        else:
            print(f'{date_start}:时分秒格式')
            # 获取当前日期
            today = datetime.date.today()
            # 获取前一天的日期
            yesterday = today - datetime.timedelta(days=1)
            # 拼接起始和截止时间
            date_start = str(yesterday) + " " + date_start
            date_end = str(yesterday) + " " + date_end

        copy_exchange(origin_path_test, dst_path, date_start, date_end)
    else:
        print(f'当前环境无任务执行')


# 日常任务-同生产频点
def job_test_p():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_test_p = data.get('origin_path_test_p', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start = data.get('date_start', 'null')
    date_end = data.get('date_end', 'null')

    # 根据路径决定是否执行任务
    if origin_path_test_p:
        # 判断时间日期格式
        if check_date_format(date_start):
            print(f'{date_start}:年月日时分秒格式')
        else:
            print(f'{date_start}:时分秒格式')
            # 获取当前日期
            today = datetime.date.today()
            # 获取前一天的日期
            yesterday = today - datetime.timedelta(days=1)
            # 拼接起始和截止时间
            date_start = str(yesterday) + " " + date_start
            date_end = str(yesterday) + " " + date_end

        copy_exchange(origin_path_test_p, dst_path, date_start, date_end)
    else:
        print(f'当前环境无任务执行')


# 日常任务-新域名
def job_prod_n():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_prod_n = data.get('origin_path_prod_n', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start = data.get('date_start', 'null')
    date_end = data.get('date_end', 'null')

    # 根据路径决定是否执行任务
    if origin_path_prod_n:
        # 判断时间日期格式
        if check_date_format(date_start):
            print(f'{date_start}:年月日时分秒格式')
        else:
            print(f'{date_start}:时分秒格式')
            # 获取当前日期
            today = datetime.date.today()
            # 获取前一天的日期
            yesterday = today - datetime.timedelta(days=1)
            # 拼接起始和截止时间
            date_start = str(yesterday) + " " + date_start
            date_end = str(yesterday) + " " + date_end

        copy_exchange(origin_path_prod_n, dst_path, date_start, date_end)

    else:
        print(f'当前环境无任务执行')


# 日常任务
def job_prod():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_prod = data.get('origin_path_prod', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start = data.get('date_start', 'null')
    date_end = data.get('date_end', 'null')

    # 根据路径决定是否执行任务
    if origin_path_prod:
        # 判断时间日期格式
        if check_date_format(date_start):
            print(f'{date_start}:年月日时分秒格式')
        else:
            print(f'{date_start}:时分秒格式')
            # 获取当前日期
            today = datetime.date.today()
            # 获取前一天的日期
            yesterday = today - datetime.timedelta(days=1)
            # 拼接起始和截止时间
            date_start = str(yesterday) + " " + date_start
            date_end = str(yesterday) + " " + date_end

        copy_exchange(origin_path_prod, dst_path, date_start, date_end)

    else:
        print(f'当前环境无任务执行')


# 周任务
def job_test_weekly():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_test = data.get('origin_path_test', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start_weekly = data.get('date_start_weekly', 'null')
    date_end_weekly = data.get('date_end_weekly', 'null')

    # 根据路径决定是否执行任务
    if origin_path_test:
        # 获取当前日期
        today = datetime.date.today()
        idx = (today.weekday() + 1) % 7  # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
        # 获取上周五、六、日的日期
        fri = today - datetime.timedelta(idx + 2)
        sun = today - datetime.timedelta(idx + 1)
        sat = today - datetime.timedelta(idx)
        # 拼接周六的起始和截止时间
        date_start_o = str(fri) + " " + date_start_weekly
        date_end_o = str(sun) + " " + date_end_weekly
        # 拼接周日的起始和截止时间
        date_start_t = str(sun) + " " + date_start_weekly
        date_end_t = str(sat) + " " + date_end_weekly

        copy_exchange(origin_path_test, dst_path, date_start_o, date_end_o)
        copy_exchange(origin_path_test, dst_path, date_start_t, date_end_t)
    else:
        print(f'当前环境无任务执行')


# 周任务-同生产频点
def job_test_p_weekly():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_test_p = data.get('origin_path_test_p', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start_weekly = data.get('date_start_weekly', 'null')
    date_end_weekly = data.get('date_end_weekly', 'null')

    if origin_path_test_p:
        # 获取当前日期
        today = datetime.date.today()
        idx = (today.weekday() + 1) % 7  # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
        # 获取上周五、六、日的日期
        fri = today - datetime.timedelta(idx + 2)
        sun = today - datetime.timedelta(idx + 1)
        sat = today - datetime.timedelta(idx)
        # 拼接周六的起始和截止时间
        date_start_o = str(fri) + " " + date_start_weekly
        date_end_o = str(sun) + " " + date_end_weekly
        # 拼接周日的起始和截止时间
        date_start_t = str(sun) + " " + date_start_weekly
        date_end_t = str(sat) + " " + date_end_weekly

        copy_exchange(origin_path_test_p, dst_path, date_start_o, date_end_o)
        copy_exchange(origin_path_test_p, dst_path, date_start_t, date_end_t)
    else:
        print(f'当前环境无任务执行')

# 周任务
def job_prod_weekly():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_prod = data.get('origin_path_prod', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start_weekly = data.get('date_start_weekly', 'null')
    date_end_weekly = data.get('date_end_weekly', 'null')

    if origin_path_prod:
        # 获取当前日期
        today = datetime.date.today()
        idx = (today.weekday() + 1) % 7  # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
        # 获取上周五、六、日的日期
        fri = today - datetime.timedelta(idx + 2)
        sun = today - datetime.timedelta(idx + 1)
        sat = today - datetime.timedelta(idx)
        # 拼接周六的起始和截止时间
        date_start_o = str(fri) + " " + date_start_weekly
        date_end_o = str(sun) + " " + date_end_weekly
        # 拼接周日的起始和截止时间
        date_start_t = str(sun) + " " + date_start_weekly
        date_end_t = str(sat) + " " + date_end_weekly

        copy_exchange(origin_path_prod, dst_path, date_start_o, date_end_o)
        copy_exchange(origin_path_prod, dst_path, date_start_t, date_end_t)
    else:
        print(f'当前环境无任务执行')


# 周任务-新域名
def job_prod_n_weekly():
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取特定键值对的值
    origin_path_prod_n = data.get('origin_path_prod_n', 'null')
    dst_path = data.get('dst_path', 'null')
    date_start_weekly = data.get('date_start_weekly', 'null')
    date_end_weekly = data.get('date_end_weekly', 'null')

    if origin_path_prod_n:
        # 获取当前日期
        today = datetime.date.today()
        idx = (today.weekday() + 1) % 7  # MON = 0, SUN = 6 -> SUN = 0 .. SAT = 6
        # 获取上周五、六、日的日期
        fri = today - datetime.timedelta(idx + 2)
        sun = today - datetime.timedelta(idx + 1)
        sat = today - datetime.timedelta(idx)
        # 拼接周六的起始和截止时间
        date_start_o = str(fri) + " " + date_start_weekly
        date_end_o = str(sun) + " " + date_end_weekly
        # 拼接周日的起始和截止时间
        date_start_t = str(sun) + " " + date_start_weekly
        date_end_t = str(sat) + " " + date_end_weekly

        copy_exchange(origin_path_prod_n, dst_path, date_start_o, date_end_o)
        copy_exchange(origin_path_prod_n, dst_path, date_start_t, date_end_t)
    else:
        print(f'当前环境无任务执行')

def run_threading(job_func):
    # 多线程并行运行
    job_thread = threading.Thread(target=job_func)
    job_thread.start()


if __name__ == '__main__':
    # 读取配置文件，加载字典
    data = txt_to_dict(glob_config)
    # 获取运行时间
    # run_time = "16:04"
    run_time = data.get('run_time', 'null')
    run_time_weekly = data.get('run_time_weekly', 'null')

    # 创建调度器
    schedule_daily = schedule.Scheduler()

    # 每天定时调度日常任务
    schedule_daily.every().day.at(run_time).do(run_threading, job_test)
    schedule_daily.every().day.at(run_time).do(run_threading, job_test_p)
    schedule_daily.every().day.at(run_time).do(run_threading, job_prod)
    schedule_daily.every().day.at(run_time).do(run_threading, job_prod_n)
    # schedule.every(2).minutes.do(run_threading, job_test)  # 每两分钟
    # schedule.every(2).minutes.do(run_threading, job_prod)  # 每两分钟

    # 创建调度器
    schedule_weekly = schedule.Scheduler()

    # 每周的星期一定时调度周任务
    schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_test_weekly)
    schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_test_p_weekly)
    schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_prod_weekly)
    schedule_weekly.every().monday.at(run_time_weekly).do(run_threading, job_prod_n_weekly)

    # 立即执行所有任务
    schedule_daily.run_all()
    schedule_weekly.run_all()

    while True:
        schedule_daily.run_pending()
        schedule_weekly.run_pending()
        time.sleep(1)

config配置文件如下：

# 日常任务执行时间-北京时间
run_time=08:05

# 读取的原文件路径，默认test、test_p、prod、prod_n四个环境，若停止监控该环境，只需将该值置空
#origin_path_test=E:/python/pythonProject1/threading_test/N_TEST_20231105.pos
origin_path_test=E:/data/0925/move.pos
origin_path_test_p=E:/data/0925/move.pos
 

# 存放文件的路径
dst_path=E:/python/pythonProject1/threading_test/

# 日常的起始匹配字符串：UTC时间：2023-11-06 08:27:08和08:27:08格式均支持，date_start和date_end必须格式相同
#date_start=2023-11-06 08:27:08
date_start=12:00:00

# 日常的截止匹配字符串：UTC时间
#date_end=2023-11-06 09:27:08
date_end=23:59:59

# 周任务执行时间-北京时间
run_time_weekly=00:05

# 周末的起始匹配字符串：UTC时间
date_start_weekly=16:00:00

# 周末的截止匹配字符串：UTC时间
date_end_weekly=15:59:59