背景:天津市津南区坚持以绿色经济驱动传统经济向数字化经济转型以实现津南高质量发展，正依托物联网、云计算、三网融合等新一代信息技术，加快全区智能工业全面发展，推动产业转型升级，培育新兴产业，实现“津南制造”向“津南智造”转变。
2019津南数字制造聚焦智能制造，以原料企业工艺优化为课题，要求以异烟酸生产过程中的各参数，设计精确智能的优秀算法，提升异烟酸的收率，助力企业实现转型升级，提升行业竞争力。大赛数据提供方天津汉德威药业有限公司，为大赛提供真实生产数据，提供工艺专家的专业指导，从软硬件环境诸多方面提供大赛支撑。
使用工具是Jupyter Notebook
项目目前提供了两份数据,分别是jinnan_round1_train_20181227(训练集,用来构建模型)和jinnan_round1_testA_20181227(测试集,用来对模型准确度进行测试)
读取并观察数据
import os
import re
import time
import datetime
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgb
import xgboost as xgb
from scipy import sparse
from sklearn.svm import SVR
from sklearn.metrics import mean_squared_error
from sklearn.metrics import log_loss
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsClassifier
from sklearn.feature_selection import SelectKBest, f_classif, f_regression
from sklearn.linear_model import BayesianRidge,Lasso,LinearRegression
from sklearn.model_selection import KFold, RepeatedKFold,train_test_split,cross_val_score
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
warnings.filterwarnings("ignore")
pd.set_option('max_columns',1000)
pd.set_option('max_row',300)
#显示所有列
# pd.set_option('display.max_columns', None)
#显示所有行
# pd.set_option('display.max_rows', None)
#设置value的显示长度为100，默认为50
pd.set_option('max_colwidth',100)
train = pd.read_csv("data/jinnan_round1_train_20181227.csv",encoding="gb18030")
test = pd.read_csv("data/jinnan_round1_testA_20181227.csv",encoding="gb18030")
train.head()
# 查询列值类别较少的特征
cols = list(train.columns)
# del cols[-1]
# del cols[0]
values_cols = []
name_cols = []
for i in cols:
if len(train[i].unique()) <= 4:
values_cols.append(train[i].unique())
name_cols.append(i)
print(values_cols)
print(name_cols)
[array([300, 250, 200], dtype=int64), array([ nan, 125.]), array([405., 340., nan, 270.]), array([700, 590, 980, 470], dtype=int64), array([100, 101, 102, 103], dtype=int64), array([0.2 , 0.15, 0.12]), array([0.2, 0.1]), array([ 9. , 10. , 8. , 3.5]), array([ 5., 4., nan, 10.]), array([3.5 , 0.15, nan, 3.6 ]), array([3.5, nan, 3.6]), array([0.15, 0.03, 0.06, nan])]
['A1', 'A2', 'A3', 'A4', 'A10', 'A13', 'A18', 'A22', 'A23', 'B2', 'B3', 'B13']
# 查询缺失率超过90%的列(列中某一的值占有率达90%以上)
cols = []
good_cols = list(train.columns)
for i in train.columns:
rate =train[i].value_counts(normalize=True, dropna=False).values[0]
if rate > 0.9:
cols.append(i)
good_cols.remove(i)
print(cols)
# 将缺失率超过90%特征的删除
for i in cols:
if i in good_cols:
good_cols.remove(i)
print(good_cols)
['A1', 'A2', 'A3', 'A4', 'A13', 'A18', 'A23', 'B2', 'B3', 'B13']
['样本id', 'A5', 'A6', 'A7', 'A8', 'A9', 'A10', 'A11', 'A12', 'A14', 'A15', 'A16', 'A17', 'A19', 'A20', 'A21', 'A22', 'A24', 'A25', 'A26', 'A27', 'A28', 'B1', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11', 'B12', 'B14', '收率']
# 将收率过低的异常值删除
train_edit = train[train['收率']>=0.85]
train = train_edit[good_cols]
good_cols.remove('收率')
good_cols.remove('样本id')
test = test[good_cols]
target_train = train['收率']
train = train[good_cols]
# 将训练集和测试集合并
data = pd.concat([train,test],axis=0,ignore_index=True)
time_cols = ['A5','A7','A9','A11',
'A14','A16','A24',
'A26','B5','B7']
time_d_cols = ['A20','A28','B4','B9','B10','B11']
# 查看时间列是否有异常值
def time_check(t):
if pd.isnull(t):
pass
else:
try:
t,m,s=t.split(":")
except:
print(t)
def time_check_d(t):
if pd.isnull(t):
pass
else:
try:
sh,sm,eh,em=re.findall(r"\d+\.?\d*",t)
except:
print(t)
for i in time_cols:
data_one = data[i].apply(time_check)
print('---------')
for i in time_d_cols:
data_two = data[i].apply(time_check_d)
1900/1/21 0:00
1900/1/29 0:00
1900/1/21 0:00
1900/1/22 0:00
1900/1/9 7:00
700
1900/1/1 2:30
1900/1/12 0:00
1900/3/13 0:00
15:00-1600
19:-20:05
# 可以看出时间列类型分为上述两种,固定时间和间隔时间
# 异常值诸如:1900/1/21 0:00 默认时间为0,并将空值均赋为时间为0,上述700值认为是7个小时
# 分别对时间列进行处理,转化为具体数值(小时)
def time_edit_second(t):
try:
t,m,s=t.split(":")
except:
if t=='1990/1/9 7:00':
return 7
elif t=='1990/1/1 2:30':
return (2*3600+30*60)/3600
elif t=='700':
return 7
else:
return 0
try:
time = (int(t)*3600+int(m)*60+int(s))/3600
return time
except:
return 0
for i in time_cols:
data[i] = data[i].apply(time_edit_second)
def time_d_edit_second(t):
try:
sh,sm,eh,em = re.findall(r"\d+\.?\d*",t)
except:
if t=='15:00-1600' or t=='19:-20:05':
return 1
else:
return 0
try:
if int(sh)>int(eh):
time_d = (int(eh)*3600+int(em)*60-int(sm)*60-int(sh)*3600)/3600 + 24
else:
time_d = (int(eh)*3600+int(em)*60-int(sm)*60-int(sh)*3600)/3600
return time_d
except:
return 0
# return time_d
for i in time_d_cols:
data[i] = data[i].apply(time_d_edit_second)
# 观察转换后的时间值有无异常值
# 观察修改后的数据,讲缺失值补足
# A8列的空值均用0代替,其他列值缺失1-3个值,可以用列的平均值进行代替
data.info()
RangeIndex: 1540 entries, 0 to 1539
Data columns (total 32 columns):
A5 1540 non-null float64
A6 1540 non-null float64
A7 1540 non-null float64
A8 167 non-null float64
A9 1540 non-null float64
A10 1540 non-null int64
A11 1540 non-null float64
A12 1540 non-null float64
A14 1540 non-null float64
A15 1540 non-null float64
A16 1540 non-null float64
A17 1540 non-null float64
A19 1540 non-null int64
A20 1540 non-null float64
A21 1537 non-null float64
A22 1540 non-null float64
A24 1540 non-null float64
A25 1539 non-null object
A26 1540 non-null float64
A27 1539 non-null float64
A28 1540 non-null float64
B1 1529 non-null float64
B4 1540 non-null float64
B5 1540 non-null float64
B6 1540 non-null int64
B7 1540 non-null float64
B8 1539 non-null float64
B9 1540 non-null float64
B10 1540 non-null float64
B11 1540 non-null float64
B12 1539 non-null float64
B14 1540 non-null int64
dtypes: float64(27), int64(4), object(1)
memory usage: 385.1+ KB
data['A8'] = data['A8'].fillna(0)
list_edit = ['A21','A27','B1','B8','B12']
for i in list_edit:
data[i] = data[i].fillna(data[i].mean())
# 报错,因为A25是object类型并且有一单元值为1900/3/10 0:00,无法转换
data.loc[list(data['A25'].values).index('1900/3/10 0:00'),'A25']=0
data=pd.DataFrame(data,dtype=np.float)
data['A25'] = data['A25'].fillna(data['A25'].mean())
# 将清理后的数据进行分配
train = data[0:train.shape[0]] # 取前1390行
test = data[train.shape[0]:] # 取1390行之后的所有行
# 对特征进行分析,验证特征与目标值的相关性
figsize = 15,8
figure = plt.subplots(figsize=figsize)
selector = SelectKBest(f_classif,k=15)
selector.fit(train,target)
score = -np.log10(selector.pvalues_)
plt.bar(range(len(train.columns)), score)
plt.xticks(range(len(train.columns)), train.columns)
plt.show()
# 对特征进行筛选,去除 A6,A24,A28特征
# 观察训练集和测试集的特征值分布情况
fig = plt.subplots(figsize=(30,20))
j = 1
for cols in data.columns:
plt.subplot(4,8,j)
sns.distplot(train[cols])
sns.distplot(test[cols])
j+=1
# 去除 A6,A24,A28特征
good_columns = list(train.columns)
remove_col = ['A6','A24','A28']
for i in remove_col:
good_columns.remove(i)
train = train[good_columns]
test = test[good_columns]
x_train,x_test,y_train,y_test = train_test_split(train,target,
test_size = 0.3,random_state=0)
#封装,并对结果进行k折交叉验证
def kfold_score(alg):
kf = KFold(n_splits=5,random_state=1,shuffle=False)
alg.fit(x_train, y_train)
score = cross_val_score(alg, x_train, y_train, cv=kf)
print(score.mean())
print(alg.score(x_test, y_test))
y_pred = alg.predict(test)
return y_pred
近日略忙! 后续继续更新!