python替换缺失值，处理空值+生成图形+图形标准化

本例用到的库及方法有：json库读取json文件，pandas中value_counts计数，fillna替换缺失值，空字符串替换，counts的plot方法生成图形

plot(kind='barh',stacked=True)堆积条形图，normed_subset=count_subset.div(count_subset.sum(1),axis=0)条形规范化，总计为1

 

#文件格式为json，python内置json模块可以将json字符串转换成字典对象

import json
path='F:\workspace\python\pydata-book-master\ch02\usagov_bitly_data2012-03-16-1331923249.txt'
records=[json.loads(line) for line in open(path)]

#接下来用pandas中value_counts对时区进行计数

from pandas import DataFrame,Series
import pandas as pd;import numpy as np
frame=DataFrame(records)
frame
frame['tz'][:10]
tz_count=frame['tz'].value_counts()
tz_count[:10]

Out[12]:
America/New_York       1251
                        521
America/Chicago         400
America/Los_Angeles     382
America/Denver          191
Europe/London            74
Asia/Tokyo               37
Pacific/Honolulu         36
Europe/Madrid            35
America/Sao_Paulo        33
Name: tz, dtype: int64

 

##利用matplotlib生成一张图片
#先给缺失的时区填上一个替代值，fillna替换NA，空字符串用布尔型数组替换
clean_tz=frame['tz'].fillna('Missing')
clean_tz[clean_tz=='']='unknown'
tz_counts=clean_tz.value_counts()
tz_counts[:10]

Out[16]: In [17]:
America/New_York       1251
unknown                 521
America/Chicago         400
America/Los_Angeles     382
America/Denver          191
Missing                 120
Europe/London            74
Asia/Tokyo               37
Pacific/Honolulu         36
Europe/Madrid            35

 

#利用counts的plot方法得到一张水平条形图
tz_counts[:10].plot(kind='barh',rot=0)

 

 

#找出windows用户，假定a字段中含有windows字样即为windows用户
frame.describe()
frame.head()
#split对字符串进行切片，Series是一种类似于一维数组的对象
results=Series([x.split()[0] for x in frame.a.dropna()])
results.head()
results.value_counts()[:8]

Out[21]:
0               Mozilla/5.0
1    GoogleMaps/RochesterNY
2               Mozilla/4.0
3               Mozilla/5.0
4               Mozilla/5.0
dtype: object

results.value_counts()[:8]
Out[22]:
Mozilla/5.0                 2594
Mozilla/4.0                  601
GoogleMaps/RochesterNY       121
Opera/9.80                    34
TEST_INTERNET_AGENT           24
GoogleProducer                21
Mozilla/6.0                    5
BlackBerry8520/5.0.0.681       4
dtype: int64

#找出windows用户
#将agent缺失的数据移除
cframe=frame[frame.a.notnull()]
operating_system=np.where(cframe['a'].str.contains('Windows'),'Windows','not Windows') #注意大小写
operating_system[:5]

#按时区对操作系统进行分组
by_tz_os=cframe.groupby(['tz',operating_system])
agg_counts=by_tz_os.size().unstack().fillna(0)
agg_counts[:10]

n [29]: count_subset=agg_counts.take(indexer)[-10:]
count_subset
Out[29]:
                     Windows  not Windows
tz                                      
America/Sao_Paulo         20           13
Europe/Madrid             19           16
Pacific/Honolulu          36            0
Asia/Tokyo                35            2
Europe/London             31           43
America/Denver            59          132
America/Los_Angeles      252          130
America/Chicago          285          115
                         276          245
America/New_York         912          339

#选取最常出现的时区
indexer=agg_counts.sum(1).argsort()
indexer[:10]
#通过take截取最后10行
count_subset=agg_counts.take(indexer)[-10:]
count_subset
#生成条形图,stacked=true生成堆积条形图
count_subset.plot(kind='barh',stacked=True)

#上图不太容易区分较小分组中的windows用户，因此将各行规范化为‘总计为1’
normed_subset=count_subset.div(count_subset.sum(1),axis=0)
normed_subset.plot(kind='barh',stacked=True)