从零开始数据分析Kaggle项目——泰坦尼克号（五）

从零开始数据分析Kaggle项目—泰坦尼克号2—2.1

# title: "Kaggle项目泰坦尼克号 2__2.1"
# author: "小鱼"
# date: "2021-12-17"
import pandas as pd
import numpy as np

df = pd.read_csv("train.csv")

# 查看每个特征缺失值个数
df.isna().sum()
df.info()

RangeIndex: 891 entries, 0 to 890
Data columns (total 12 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   PassengerId  891 non-null    int64  
 1   Survived     891 non-null    int64  
 2   Pclass       891 non-null    int64  
 3   Name         891 non-null    object 
 4   Sex          891 non-null    object 
 5   Age          714 non-null    float64
 6   SibSp        891 non-null    int64  
 7   Parch        891 non-null    int64  
 8   Ticket       891 non-null    object 
 9   Fare         891 non-null    float64
 10  Cabin        204 non-null    object 
 11  Embarked     889 non-null    object 
dtypes: float64(2), int64(5), object(5)
memory usage: 83.7+ KB

查看指定列数据

# 查看Age， Cabin， Embarked列的数据
df[['Age','Cabin','Embarked']].head(6)

	Age	Cabin	Embarked
0	22.0	NaN	S
1	38.0	C85	C
2	26.0	NaN	S
3	35.0	C123	S
4	35.0	NaN	S
5	NaN	NaN	Q

对缺失值处理

# # 对缺失值进行处理汇总，面对缺失值三种处理方法：
# option 1： 去掉含有缺失值的样本（行）
# option 2：将含有缺失值的列（特征向量）去掉
# option 3：将缺失值用某些值填充（0，平均值，中值等）

# df.dropna()   #删除缺失值
# df.fillna()   #填充缺失值
# df.isna()     #判断缺失值
# df.notna()    #判断缺失值

df[df['Age']==None]=0 

df[df['Age'].isna()] = 0

df.info()

RangeIndex: 891 entries, 0 to 890
Data columns (total 12 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   PassengerId  891 non-null    int64  
 1   Survived     891 non-null    int64  
 2   Pclass       891 non-null    int64  
 3   Name         891 non-null    object 
 4   Sex          891 non-null    object 
 5   Age          891 non-null    float64
 6   SibSp        891 non-null    int64  
 7   Parch        891 non-null    int64  
 8   Ticket       891 non-null    object 
 9   Fare         891 non-null    float64
 10  Cabin        362 non-null    object 
 11  Embarked     889 non-null    object 
dtypes: float64(2), int64(5), object(5)
memory usage: 83.7+ KB

# DataFrame.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)
# axis:
# axis=0: 删除包含缺失值的行
# axis=1: 删除包含缺失值的列
# how: 与axis配合使用
# how=‘any’ :只要有缺失值出现，就删除该行货列
# how=‘all’: 所有的值都缺失，才删除行或列
# thresh： axis中至少有thresh个非缺失值，否则删除
# 比如 axis=0，thresh=10：标识如果该行中非缺失值的数量小于10，将删除改行
# subset: list
# 在哪些列中查看是否有缺失值
# inplace: 是否在原数据上操作。如果为真，返回None否则返回新的copy，去掉了缺失值

df.isna().sum()

PassengerId      0
Survived         0
Pclass           0
Name             0
Sex              0
Age              0
SibSp            0
Parch            0
Ticket           0
Fare             0
Cabin          529
Embarked         2
dtype: int64

#删除包含缺失值的行
# df1 = df.dropna(axis = 0)
# df1.isna().sum()

#指定列
df1 = df.dropna(subset=['Cabin', 'Embarked'])
df1.isna().sum()
df1.info()

Int64Index: 360 entries, 1 to 889
Data columns (total 12 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   PassengerId  360 non-null    int64  
 1   Survived     360 non-null    int64  
 2   Pclass       360 non-null    int64  
 3   Name         360 non-null    object 
 4   Sex          360 non-null    object 
 5   Age          360 non-null    float64
 6   SibSp        360 non-null    int64  
 7   Parch        360 non-null    int64  
 8   Ticket       360 non-null    object 
 9   Fare         360 non-null    float64
 10  Cabin        360 non-null    object 
 11  Embarked     360 non-null    object 
dtypes: float64(2), int64(5), object(5)
memory usage: 36.6+ KB

# DataFrame.fillna(value=None, method=None, axis=None, inplace=False, limit=None, downcast=None)
# value   scalar, dict, Series, or DataFrame
# dict  可以指定每一行或列用什么值填充
# method   {‘backfill’, ‘bfill’, ‘pad’, ‘ffill’, None}, default None
# 在列上操作
# ffill / pad   使用前一个值来填充缺失值
# backfill / bfill   使用后一个值来填充缺失值
# limit 填充的缺失值个数限制

df.isna().sum()

PassengerId      0
Survived         0
Pclass           0
Name             0
Sex              0
Age              0
SibSp            0
Parch            0
Ticket           0
Fare             0
Cabin          529
Embarked         2
dtype: int64

#用0代替所有的缺失值
df2 = df.fillna(value=0)
df2.isna().sum()

PassengerId    0
Survived       0
Pclass         0
Name           0
Sex            0
Age            0
SibSp          0
Parch          0
Ticket         0
Fare           0
Cabin          0
Embarked       0
dtype: int64

判断重复值

# 判断重复值
df3 = df[df.duplicated()]  #没有参数，要全部一样才会判断重复值
df3

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked
17	0	0	0	0	0	0.0	0	0	0	0.0	0	0
19	0	0	0	0	0	0.0	0	0	0	0.0	0	0
26	0	0	0	0	0	0.0	0	0	0	0.0	0	0
28	0	0	0	0	0	0.0	0	0	0	0.0	0	0
29	0	0	0	0	0	0.0	0	0	0	0.0	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...
859	0	0	0	0	0	0.0	0	0	0	0.0	0	0
863	0	0	0	0	0	0.0	0	0	0	0.0	0	0
868	0	0	0	0	0	0.0	0	0	0	0.0	0	0
878	0	0	0	0	0	0.0	0	0	0	0.0	0	0
888	0	0	0	0	0	0.0	0	0	0	0.0	0	0

176 rows × 12 columns

# 对重复值进行处理
df3 = df.drop_duplicates()  #删除数据记录中所有列值相同的记录
df3 

df4 = df.drop_duplicates(['Age','Parch'])    #删除数据记录中指定列值相同的记录
df4

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked
0	1	0	3	Braund, Mr. Owen Harris	male	22.00	1	0	A/5 21171	7.2500	NaN	S
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.00	1	0	PC 17599	71.2833	C85	C
2	3	1	3	Heikkinen, Miss. Laina	female	26.00	0	0	STON/O2. 3101282	7.9250	NaN	S
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.00	1	0	113803	53.1000	C123	S
5	0	0	0	0	0	0.00	0	0	0	0.0000	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...
831	832	1	2	Richards, Master. George Sibley	male	0.83	1	1	29106	18.7500	NaN	S
843	844	0	3	Lemberopolous, Mr. Peter L	male	34.50	0	0	2683	6.4375	NaN	C
851	852	0	3	Svensson, Mr. Johan	male	74.00	0	0	347060	7.7750	NaN	S
871	872	1	1	Beckwith, Mrs. Richard Leonard (Sallie Monypeny)	female	47.00	1	1	11751	52.5542	D35	S
879	880	1	1	Potter, Mrs. Thomas Jr (Lily Alexenia Wilson)	female	56.00	0	1	11767	83.1583	C50	C

177 rows × 12 columns

df.to_csv('test_clear.csv')

分箱操作

#特征观察与处理
# 数值型特征一般可以直接用于模型的训练，但有时候为了模型的稳定性及鲁棒性会对连续变量进行离散化。文本型特征往往需要转换成数值型特征才能用于建模分析

#分箱操作:连续数据的离散化处理+
# 将连续变量Age平均分箱成5个年龄段，并分别用类别变量12345表示
df['AgeBand'] = pd.cut(df['Age'], 5 ,labels=[1,2,3,4,5])
df

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked	AgeBand
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	NaN	S	2
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C85	C	3
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	NaN	S	2
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	C123	S	3
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	NaN	S	3
...	...	...	...	...	...	...	...	...	...	...	...	...	...
886	887	0	2	Montvila, Rev. Juozas	male	27.0	0	0	211536	13.0000	NaN	S	2
887	888	1	1	Graham, Miss. Margaret Edith	female	19.0	0	0	112053	30.0000	B42	S	2
888	0	0	0	0	0	0.0	0	0	0	0.0000	0	0	1
889	890	1	1	Behr, Mr. Karl Howell	male	26.0	0	0	111369	30.0000	C148	C	2
890	891	0	3	Dooley, Mr. Patrick	male	32.0	0	0	370376	7.7500	NaN	Q	2

891 rows × 13 columns

df

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked	AgeBand
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	NaN	S	2
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C85	C	3
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	NaN	S	2
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	C123	S	3
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	NaN	S	3
...	...	...	...	...	...	...	...	...	...	...	...	...	...
886	887	0	2	Montvila, Rev. Juozas	male	27.0	0	0	211536	13.0000	NaN	S	2
887	888	1	1	Graham, Miss. Margaret Edith	female	19.0	0	0	112053	30.0000	B42	S	2
888	0	0	0	0	0	0.0	0	0	0	0.0000	0	0	1
889	890	1	1	Behr, Mr. Karl Howell	male	26.0	0	0	111369	30.0000	C148	C	2
890	891	0	3	Dooley, Mr. Patrick	male	32.0	0	0	370376	7.7500	NaN	Q	2

891 rows × 13 columns

# 将连续变量Age划分为(0,5] (5,15] (15,30] (30,50] (50,80]五个年龄段，并分别用类别变量12345表示
df['AgeBand1'] = pd.cut(df['Age'],[0,5,15,30,50,80],labels = [1,2,3,4,5]) 
df

#将连续变量Age按10% 30% 50 70% 90%五个年龄段，并用分类变量12345表示
# df['AgeBand2'] = pd.qcut(df['Age'],[0,0.1,0.3,0.5,0.7,0.9],labels = [1,2,3,4,5])
# df

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked	AgeBand	AgeBand1
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	NaN	S	2	3
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C85	C	3	4
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	NaN	S	2	3
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	C123	S	3	4
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	NaN	S	3	4
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
886	887	0	2	Montvila, Rev. Juozas	male	27.0	0	0	211536	13.0000	NaN	S	2	3
887	888	1	1	Graham, Miss. Margaret Edith	female	19.0	0	0	112053	30.0000	B42	S	2	3
888	0	0	0	0	0	0.0	0	0	0	0.0000	0	0	1	NaN
889	890	1	1	Behr, Mr. Karl Howell	male	26.0	0	0	111369	30.0000	C148	C	2	3
890	891	0	3	Dooley, Mr. Patrick	male	32.0	0	0	370376	7.7500	NaN	Q	2	4

891 rows × 14 columns

# 对文本变量进行转换
# 查看类别文本变量名及种类
df['Sex'].value_counts()     # value_counts() 

male      453
female    261
0         177
Name: Sex, dtype: int64

df['Sex'].unique()    #unique
df['Sex'].nunique()   #文本变量名数量

3

# 将类别文本转换为12345
df['Sex_num'] = df['Sex'].replace(['male','female'],[1,2])   # replace替换类别文本一
df['Sex_num'].value_counts() 

1    453
2    261
0    177
Name: Sex_num, dtype: int64

df['Sex_num'] = df['Sex'].map({'male': 1, 'female': 2})  # map替换类别文本二
df['Sex_num'].value_counts() 

1.0    453
2.0    261
Name: Sex_num, dtype: int64

from sklearn.preprocessing import LabelEncoder  #使用sklearn.preprocessing的LabelEncoder替换类别文本三
for feat in ['Cabin', 'Ticket']:
    lbl = LabelEncoder()  
    label_dict = dict(zip(df[feat].unique(), range(df[feat].nunique())))
    df[feat + "_labelEncode"] = df[feat].map(label_dict)
    df[feat + "_labelEncode"] = lbl.fit_transform(df[feat].astype(str))
df.head(5)

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked	AgeBand	AgeBand1	Sex_num	Cabin_labelEncode	Ticket_labelEncode
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	NaN	S	2	3	1.0	135	409
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	C85	C	3	4	2.0	74	472
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	NaN	S	2	3	2.0	135	533
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	C123	S	3	4	2.0	50	41
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	NaN	S	3	4	1.0	135	374

# 将文本变量Sex， Cabin， Embarked用one-hot编码表示
#将类别文本转换为one-hot编码

for feat in ["Age", "Embarked"]:             # OneHotEncoder
#     x = pd.get_dummies(df["Age"] // 6)  
#     x = pd.get_dummies(pd.cut(df['Age'],5))
    x = pd.get_dummies(df[feat], prefix=feat)
    df = pd.concat([df, x], axis=1)
    #df[feat] = pd.get_dummies(df[feat], prefix=feat)
    
df.head(5)

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	...	Age_66.0	Age_70.0	Age_70.5	Age_71.0	Age_74.0	Age_80.0	Embarked_0	Embarked_C	Embarked_Q	Embarked_S
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	...	0	0	0	0	0	0	0	0	0	1
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	...	0	0	0	0	0	0	0	1	0	0
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	...	0	0	0	0	0	0	0	0	0	1
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	...	0	0	0	0	0	0	0	0	0	1
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	...	0	0	0	0	0	0	0	0	0	1

5 rows × 110 columns

提取特征，正则表达式

# 从纯文本Name特征里提取出Titles的特征(所谓的Titles就是Mr,Miss,Mrs等)
# Series.str.extract(pat, flags=0, expand=True)
# 小括号表示要提取的值，中括号里面表示任意大小写字母，中括号后面加号表示，任意大小写字母任意数量，方斜杠点表示匹配点。整个正则表达式没有^，表示从任意位置开始，expand=False表示取匹配到的第一个结果。整体效果：提取点之前任意数量的大小写字母组合。
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)   # str.extract()函数和正则表达式，可以处理数字、符号和字母混合的字符串
df.head(5)

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	...	Age_70.0	Age_70.5	Age_71.0	Age_74.0	Age_80.0	Embarked_0	Embarked_C	Embarked_Q	Embarked_S	Title
0	1	0	3	Braund, Mr. Owen Harris	male	22.0	1	0	A/5 21171	7.2500	...	0	0	0	0	0	0	0	0	1	Mr
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38.0	1	0	PC 17599	71.2833	...	0	0	0	0	0	0	1	0	0	Mrs
2	3	1	3	Heikkinen, Miss. Laina	female	26.0	0	0	STON/O2. 3101282	7.9250	...	0	0	0	0	0	0	0	0	1	Miss
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35.0	1	0	113803	53.1000	...	0	0	0	0	0	0	0	0	1	Mrs
4	5	0	3	Allen, Mr. William Henry	male	35.0	0	0	373450	8.0500	...	0	0	0	0	0	0	0	0	1	Mr

5 rows × 111 columns

df.to_csv('test_fin.csv')

本章共四节，本章第2.1节主要内容，包括数据的清洗及特征处理，缺失值和重复值的处理，连续数据的离散化，转换类别文本，正则表达式。