机器学习之数据预处理——降噪

机器学习之数据预处理——降噪

上一节学习线性回归法填补缺失值和拉格朗日插值法，这节课学习采用等深分箱的方式进行数据降噪处理。

1.降噪方法

money=[800,1000,1200,1500,1600,1800,2000,2300,\
       2500,2800,3000,3500,4000,4500,4800,5000]
cut1=pd.cut(pd.Series(money), bins=[0,1000,2000,3000,4000,5000])#设定分箱区间
#0可以写，也可以不写
print(pd.value_counts(cut1))

cut3=pd.qcut(pd.Series(money), 4)#设定分箱数，每组数据量相同
print(pd.value_counts(cut3))

2.分箱平滑

#平滑噪声—等深分箱—均值平滑
import pandas as pd
import numpy as np
def aequilatus_box_mean(data,bins):
    length=data.shape[0]
    labels=[]
    for i in range(bins):
        labels.append('a'+str(i+1))#添加标签
    new_data=pd.qcut(data.iloc[:,0],bins,labels=labels)#等深分箱
    data['label']=new_data
    for label in labels:
        label_index_min=data[data.label==label].index.min()#分箱后索引最小值
        label_index_max=data[data.label==label].index.max()#分箱后索引最大值
        data.loc[label_index_min:label_index_max,data.columns[0]]=np.mean(
            data.A[label_index_min:label_index_max+1,])#根据label及索引，修改A为各箱均值
    return data

if __name__=="__main__":
    data=pd.DataFrame({'A':[11,13,15,20,20,23,26,29,35]})
    bins=3
    print("均值平滑")
    print(aequilatus_box_mean(data,3))

#平滑噪声—等深分箱—中值平滑
import pandas as pd
import numpy as np
def aequilatus_box_median(data,bins):
    length=data.shape[0]
    labels=[]
    for i in range(bins):
        labels.append('a'+str(i+1))
    new_data=pd.qcut(data.A,bins,labels=labels)#等深分箱
    data['label']=new_data
    for label in labels:
        label_index_min=data[data.label==label].index.min()#分箱后索引最小值
        label_index_max=data[data.label==label].index.max()#分箱后索引最大值
        data.loc[label_index_min:label_index_max,'A']=np.median(
            data.A[label_index_min:label_index_max+1,])#根据label及索引，修改A为各箱均值
    return data
if __name__=="__main__":
    data=pd.DataFrame({'A':[11,13,15,20,20,23,26,29,35]})
    bins=3
    print("中值平滑")
    print(aequilatus_box_median(data,3))

#平滑噪声—等深分箱—边界平滑
import pandas as pd
import numpy as np
def aequilatus_box_border(data,bins):
    length=data.shape[0]
    labels=[]
    for i in range(bins):
        labels.append('a'+str(i+1))
    new_data=pd.qcut(data.A,bins,labels=labels)#等深分箱
    data['label']=new_data
    for label in labels:
        label_index_min=data[data.label==label].index.min()
        label_index_max=data[data.label==label].index.max()
        data_min=np.min(data.A[label_index_min:label_index_max+1,])
        data_max=np.max(data.A[label_index_min:label_index_max+1,])
        for i in range(label_index_min,label_index_max): 
            if(data.loc[i,'A']==data_min or data.loc[i,'A']==data_max):
                data.loc[i,'A']=data.loc[i,'A']
            elif(np.abs(data.loc[i,'A']-data_min)<=np.abs(data.loc[i,'A']-data_max)):
                 data.loc[i,'A']=data_min
            else:
                data.loc[i,'A']=data_max
    return data   
if __name__=="__main__":
    data=pd.DataFrame({'A':[11,12,15,21,20,23,26,29,35]})
    bins=3
    print("边界平滑")
    print(aequilatus_box_border(data,3))

编写打磨课件不易，走过路过别忘记给咱点个赞，小女子在此(❁´ω`❁)谢过！如需转载，请注明出处，Thanks♪(･ω･)ﾉ

参考文献：

1.https://blog.csdn.net/weixin_40192436/article/details/86706231

2.https://www.cnblogs.com/serena45/p/5559122.html

3.https://www.jianshu.com/p/389682aa5429