003 pandas 数据清洗常用

操作数据库实例

import pandas as pd
from sqlalchemy import create_engine

conn = create_engine(
    'mysql+pymysql://root:xxxxxx@localhost:3306/testdb?charset=utf8')

# 从数据库读取
# data = pd.read_sql('select * from company_info limit 100', conn)
data = pd.read_sql_table('company_info_xian', conn)

# 去重，可以传递列表，按照多个字段去重
data.drop_duplicates(subset='phone', inplace=True)

# 保存到数据库中
data.to_sql('company_xian_no_duplicates', conn, index=False, chunksize=100)