Python Excel导入Mysql的通用方法

---

一、前言

此代码将导入部分尽量通用，仅配置下面两项就可以进行导入了：

1. 从哪个excel导入到哪个mysql表
   在程序中配置
   

2. 他们之间的字段如何对应
   写在mysql表中
   
   ps：id，create_time，update_time 这种常用字段写死在程序中即可

---

二、实现

ps：程序中的 mysql导入方法可以用pymysql自己实现或者参考此处
https://blog.csdn.net/weixin_43721000/article/details/124803077

import datetime
import json

from utils.MysqlOp import MySqlOp
from config import Config as cf
import pandas as pd
import numpy as np


def isNaNo(sth):
    '''
    NaN、None或者空字符串返回True，其他情况返回False
    '''
    if not sth:
        return True
    if isinstance(sth, float):
        if np.isnan(sth):
            return True
    return False


def importExcelToMysql(excel_path, mysql_table):
    '''
    excel表导入mysql
    :param excel_path:
    :param mysql_table:
    :return:
    '''

    # 读取excel
    df = pd.read_csv(excel_path, dtype=str)

    # 创建mysql连接
    msq = MySqlOp(host=cf.MYSQL_HOST, port=cf.MYSQL_PORT, username=cf.MYSQL_USER, password=cf.MYSQL_PWD, db=cf.MYSQL_DB)

    # 执行sql
    data_field_list = msq.select(f'''
                show full columns from `{mysql_table}`;
            ''')
    print(data_field_list)


    sql_list = []

    for i in range(df.shape[0]):

        print(f'当前进度： {i + 1} / {df.shape[0]}')

        mysql_field_list = []
        excel_value_list = []
        for data_field in data_field_list:

            # mysql的字段名
            mysql_field = data_field[0]
            # mysql的字段类型
            mysql_field_type = data_field[1]
            # excel的字段名存在mysql的字段注释中
            excel_field = data_field[-1]

            # 跳过excel字段不存在的情况
            if not excel_field:
                continue
            # 跳过注释与excel字段对应不上的情况
            if excel_field not in df.columns.tolist():
                continue

            # 获取excel每行字段对应的值
            excel_value = df.loc[i, excel_field]
            if isNaNo(excel_value):
                excel_value = 'null'
            elif mysql_field_type.startswith(("text", "varchar", "datetime")):
                excel_value = json.dumps(excel_value, ensure_ascii=False)

            # print(mysql_field, excel_field, excel_value)
            mysql_field_list.append(mysql_field)
            excel_value_list.append(excel_value)

        sql = f'''
                INSERT INTO `{mysql_table}`
                ({', '.join(mysql_field_list)}, {"create_time"}, {"update_time"})
                VALUES
                ({', '.join(excel_value_list)}, {json.dumps(str(datetime.datetime.now()), ensure_ascii=False)}, {json.dumps(str(datetime.datetime.now()), ensure_ascii=False)})
            '''

        sql_list.append(sql)

        if (i % 100 == 0 and i != 0) or (i+1 == df.shape[0]):
            op_id_list = msq.batch_operate(sql_list)
            print(f'op_id_list: {op_id_list}')
            print(f'len: {len(op_id_list)}')
            sql_list = []

        # # print(sql)
        # msq.operate(sql)


if __name__ == '__main__':

    excel_path = r'E:\oa书\1\总详情.xlsx'       # 从哪里导
    mysql_table = 'data'                       # 导哪里去

    importExcelToMysql(excel_path, mysql_table) # 导