1 导入sklearn的分类库
# 通用算法模型
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from xgboost import XGBClassifier
# 通用帮助模型
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
# 可视化模块
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
from pandas.tools.plotting import scatter_matrix
# 数据操作
import numpy as np
import pandas as pd
color = sns.color_palette()
# 配置可视化的默认值
%matplotlib inline
mpl.style.use('ggplot')
sns.set_style('white')
pylab.rcParams['figure.figsize'] = 12, 8
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
2 数据读入
df1 = pd.read_csv("./data/20180506/1.dat")
df2 = pd.read_csv("./data/20180506/2.dat")
df3 = pd.read_csv("./data/20180506/3.dat")
df4 = pd.read_csv("./data/20180506/4.dat")
df5 = pd.read_csv("./data/20180506/5.dat")
data = df5.append(df4)
data = data.append(df3)
data = data.append(df2)
data = data.append(df1)
data.columns = ['c_num_pack', 's_num_pack', 'total_num_pack', 'c_pack_size_expec', 's_pack_size_expec', 'c_pack_per_sec','s_pack_per_sec', 'c_pack_size_var', 's_pack_size_var', 'total_c_bytes', 'total_s_bytes', 'c_bytes_per_sec','s_bytes_per_sec','down_to_up_ratio', 'protocal']
print(data.info())
查看一下数字
数据概述
3 数据整理
# Converting to categorical data
Target = ['protocal']
data1_x = ['c_num_pack', 's_num_pack', 'total_num_pack', 'c_pack_size_expec',
's_pack_size_expec', 'c_pack_per_sec', 's_pack_per_sec',
'c_pack_size_var', 's_pack_size_var', 'total_c_bytes',
'total_s_bytes', 'c_bytes_per_sec', 's_bytes_per_sec',
'down_to_up_ratio']
data1_xy = Target + data1_x
print('Original X Y', data1_xy, '\n')
train1_x, test1_x, train1_y, test1_y = model_selection.train_test_split(data[data1_x], data[Target], random_state=0)
print('Datal Shape: {}'.format(data.shape))
train1_x.head()
image.png
4 数据探索
# Important: Intentionally plotted different ways for learning purposes only
# optional plotting w/pandas
#
plt.figure(figsize=(16, 12))
plt.subplot(231)
plt.boxplot(x=data['c_num_pack'], showmeans=True, meanline=True)
plt.title('client_num_pack Boxplot')
plt.ylabel('nums (num)')
plt.subplot(232)
plt.boxplot(data['s_num_pack'], showmeans=True, meanline=True)
plt.title('server_num_packets Boxplot')
plt.ylabel('Packets (num)')
plt.subplot(233)
plt.boxplot(data['total_num_pack'],showmeans=True, meanline=True)
plt.ylabel('total num packets')
plt.subplot(234)
plt.hist(x=[data[data['protocal']==1]['total_num_pack'], data[data['protocal']==1]['down_to_up_ratio']], stacked=True, color=['g', 'r'], label=['packets', 'ratio'])
plt.title('protocal 1 ')
plt.xlabel('num {packets}')
plt.ylabel('ratio')
plt.legend()
'''
plt.subplot(235)
plt.hist(x=[data1[data1['Survived']==1]['Age'], data1[data1['Survived']==0]['Age']], stacked=True, color=['g', 'r'], label=['Survived', 'Dead'])
plt.title('Age Histogram by Survival')
plt.xlabel('Age (Years)')
plt.ylabel('# of Passengers')
plt.legend()
plt.subplot(236)
plt.hist(x=[data1[data1['Survived']==1]['FamilySize'], data1[data1['Survived']==0]['FamilySize']], stacked=True, color=['g', 'r'], label=['Survived', 'Dead'])
plt.title('Family Size Histogram by Survival')
plt.xlabel('Family Size(#)')
plt.ylabel('# of Passengers')
plt.legend()
exploration
f, ax = plt.subplots(1, 2, figsize=(20, 10))
data1 = data.copy(deep=True)
data1['protocol_type'] = 'unkowntype'
for i, t in enumerate(['ICMP', 'UDP', 'SMTP', 'POP3', 'IMAP', 'HTTP', 'TCP-NC', 'FTP', 'SSH']):
data1.loc[data1.protocal == i+1, 'protocol_type'] = t
data1['protocol_type'].value_counts().plot.pie(autopct='%1.1f%%', ax=ax[0], shadow=True)
ax[0].set_title('protocal')
ax[0].set_ylabel('')
sns.countplot('protocol_type', data=data1, ax=ax[1])
ax[1].set_title('protocol')
plt.show()
image.png
f, ax = plt.subplots(1, 2, figsize=(20, 10))
sns.violinplot('total_num_pack', 'total_c_bytes', hue='protocol_type', data=data1, ax=ax[0])
sns.violinplot('total_num_pack', 'total_s_bytes', hue='protocol_type', data=data1, ax=ax[1])
image.png
5 开始跑模型
# Machine Learning Algorithm (MLA) Selection and Initialization
MLA = [
# Ensemble Methods
ensemble.AdaBoostClassifier(),
ensemble.BaggingClassifier(),
ensemble.ExtraTreesClassifier(),
ensemble.GradientBoostingClassifier(),
ensemble.RandomForestClassifier(),
# Gaussian Processes
# gaussian_process.GaussianProcessClassifier(),
#GLM
linear_model.LogisticRegressionCV(),
linear_model.PassiveAggressiveClassifier(),
linear_model.RidgeClassifierCV(),
linear_model.SGDClassifier(),
linear_model.Perceptron(),
#Navies Bayes
naive_bayes.BernoulliNB(),
naive_bayes.GaussianNB(),
#Nearest Neighbor
neighbors.KNeighborsClassifier(),
# SVM
svm.SVC(probability=True),
# svm.NuSVC(probability=True),
svm.LinearSVC(),
# Trees
tree.DecisionTreeClassifier(),
tree.ExtraTreeClassifier(),
#Discriminant Analysis
discriminant_analysis.LinearDiscriminantAnalysis(),
discriminant_analysis.QuadraticDiscriminantAnalysis(),
XGBClassifier()
]
cv_split = model_selection.ShuffleSplit(n_splits=10, test_size=.3, train_size=.6, random_state=0)
MLA_columns = ['MLA Name', 'MLA Parameters', 'MLA Train Accuracy Mean', 'MLA Test Accuracy Mean', 'MLA Test Accuracy 3*STD', 'MLA Time']
MLA_compare = pd.DataFrame(columns=MLA_columns)
MLA_predict = data[Target]
row_index = 0
for i, alg in enumerate(MLA):
print(i, alg.__class__.__name__)
MLA_name = alg.__class__.__name__
MLA_compare.loc[row_index, 'MLA Name'] = MLA_name
MLA_compare.loc[row_index, 'MLA Parameters'] = str(alg.get_params())
cv_results = model_selection.cross_validate(alg, data[data1_x], data[Target].values.ravel(), cv = cv_split, return_train_score=True)
MLA_compare.loc[row_index, 'MLA Time'] = cv_results['fit_time'].mean()
MLA_compare.loc[row_index, 'MLA Train Accuracy Mean'] = cv_results['train_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy Mean'] = cv_results['test_score'].mean()
MLA_compare.loc[row_index, 'MLA Test Accuracy 3*STD'] = cv_results['test_score'].std()*3
alg.fit(data[data1_x], data[Target].values.ravel())
MLA_predict[MLA_name] = alg.predict(data[data1_x])
row_index += 1
MLA_compare.sort_values(by=['MLA Test Accuracy Mean'], ascending=False, inplace=True)
MLA_compare
6 准确度排序
accurate
7 输入相关分析
def correlation_heatmap(df):
_, ax = plt.subplots(figsize=(14, 12))
colormap = sns.diverging_palette(220, 10, as_cmap=True)
_ = sns.heatmap(
df.corr(),
cmap=colormap,
square=True,
ax=ax,
annot=True,
linewidths=0.1,
vmax=1.0, linecolor='white',
annot_kws={'fontsize':12}
)
plt.title('Pearson Correlation of Feature', y=0.05, size=15)
correlation_heatmap(data)
relation