[转] R 代码 00003 18.06.19

# 知识来源于网络，仅供交流使用，如有侵权请及时联系予以删除

# Libraries
library(data.table)   # 高效数据操作
library(magrittr)  # 管道操作
library(ggplot2)  # 数据可视化
library(stringr)   # 字符串处理
#  library(quanteda)  该包在加载时出现错误
library(gridExtra)  # 多图
library(dplyr)    # 数据操作
library(tidyr)     # 数据操作
library(caTools)  # 工具：移动窗口统计
library(xgboost)  # 极限梯度提升
library(quanteda)   # 文本数据的定量分析
library(SnowballC)  # 基于C libstemmer UTF-8库的雪球词干分析器
library(tm)  # 文本挖掘软件包
library(corrplot)      # 相关矩阵的可视化


# Data Overview
setwd("e:/")
system.time(train <- fread('../input/train.tsv', showProgress = T , data.table=F))
    # 读取数据，包括工具条、读取时间
str(train)
    # train_id、name、item_condition_id、category_name、brand_name、price、shipping、item_description
dim(train)   # 记录多少
print(object.size(train), units = 'Mb')  # 数据存储大小

# 0: Variable Analysis：Price :价格、及其分布
length(train$price[train$price==""])
length(train$price[is.na(train$price)])
range(train$price)
ggplot(train,aes(x=price))+geom_histogram(fill = 'orangered2')  # 分布范围大，但是不均衡，变换log()表示
ggplot(data = train, aes(x = log(price+1,base=10))) + geom_histogram(fill = 'orangered2')
    # e = 2.718281828459; log(8,2)===>3;   base=exp(1)，即e

# 1: Variable Analysis：item_condition_id ：产品状况分类情况、及其对价格的影响length(train$item_condition_id[train$item_condition_id==""])
length(train$item_condition_id[is.na(train$item_condition_id)])
table(train$item_condition_id)  # 查看分类分布、与价格关系p1<-train %>%        # 画柱状图
	group_by(item_condition_id) %>%
	summarise(count=length(price),median=median(price))  %>%
	ggplot(aes(x = item_condition_id, y = count)) +  geom_bar(stat = 'identity',fill = "orangered2") 
p2<-train %>%	     # 画箱体图
	ggplot(aes(x = as.factor(item_condition_id), y = log(price+1,base=10))) + 
		stat_boxplot(geom = "errorbar") +  geom_boxplot(fill = "skyblue")  
grid.arrange(p1,p2,nrow=1)
    # 以下为箱体图的解读样本

# 2：Variable Analysis：Shipping ：运费状况，及对价格分布的影响
length(train$shipping[train$shipping==""])
length(train$shipping[is.na(train$shipping)])
table(train$shipping) # 分布状况
train %>% 
    ggplot(aes(x = log(price+1), fill = as.factor(shipping))) + 
    geom_density(adjust=2,alpha= 0.6)

# 3：Variable Analysis：brand_name ：品牌名称，及对价格分布的影响
length(train$brand_name[train$brand_name==""])
length(train$brand_name[is.na(train$brand_name)])
length(table(train$brand_name))  # 分布状况
train %>%  
    group_by(brand_name) %>%     
    summarise(median_price = median(price)) %>%         
    arrange(desc(median_price)) %>% head(25) %>%        
    ggplot(aes(x = reorder(brand_name,median_price), y = median_price)) +         
        geom_point()+coord_flip()

# 4：Variable Analysis：category_name ：产品分类名称，及对价格分布的影响
length(train$category_name[train$category_name==""])
length(train$category_name[is.na(train$category_name)])
length(unique(train$category_name))
      # 等价于  length(table(train$category_name))  # 分布状况
sort(table(train$category_name), decreasing = TRUE)[1:10]
    #分类初始分析
        train %>%  
            group_by(category_name) %>%      
            summarise(median_price = median(price)) %>%             
            arrange(desc(median_price)) %>% head(25) %>%
            ggplot(aes(x = reorder(category_name,median_price), y = median_price)) +         
            geom_point()+coord_flip()

    # 分类分析，进一步细分
        splitVar = str_split(train$cat, "/")
        cat1 = sapply(splitVar,'[',1)
        cat2 = sapply(splitVar,'[',2)
        train['cat1'] = cat1
        train['cat2'] = cat2
        train$cat1[is.na(train$cat2)] = -1
        train$cat2[is.na(train$cat3)] = -1
        train['train$category_name'][is.na(train$train$category_name)] = -1
        # cat1  分析1
                train %>%  ggplot(aes(x = cat1, y = log(price+1,base=10))) + stat_boxplot(geom = "errorbar")+
		    geom_boxplot(fill = 'cyan2', color = 'darkgrey') + coord_flip() + labs(y="",title = 'category_name: cat1 观察方法1' )
        # cat1  分析2
	        p1 <-train %>%
                        group_by(cat1, item_condition_id) %>%
                        summarise(count=length(train_id)) %>%
                        ggplot(aes(x = item_condition_id, y = cat1, fill = count/1000)) +geom_tile() +
                            scale_fill_gradient(low = 'lightblue', high = 'cyan4') +
                            labs(x = 'Condition', y = '', fill = 'Number of items (000s)', title = 'cat1: Item count by category and condition') +
               	            theme_bw() +  theme(legend.position = 'bottom')
	        p2 <-train %>%
		        group_by(cat1, item_condition_id) %>%
		        summarise(median_price=median(price)) %>%
		        ggplot(aes(x = item_condition_id, y = cat1, fill = median_price)) +
			    geom_tile() + scale_fill_gradient(low = 'lightblue', high = 'cyan4') +
			    labs(x = 'Condition', y = '', fill = 'median_price', title = 'cat1: Item price by category and condition') +
			    theme_bw() + theme(legend.position = 'bottom', axis.text.y = element_blank())
	        grid.arrange(p1, p2, ncol = 2)
        # cat2  分析
            ss<- train  %>% group_by(cat2) %>%summarise(median=median(price)) %>% arrange(desc(median)) %>% head(15)
            train %>%  filter(cat2 %in% ss$cat2) %>% select(c("price","cat1","cat2","category_name")) %>%
	        ggplot(aes(x = cat2, y = log(price+1))) + stat_boxplot(geom = "errorbar") + 
		geom_boxplot(fill = 'cyan2', color = 'darkgrey') + coord_flip()

# 5：Variable Analysis：item_despription ：产品分描述
        train['desclength'] = str_length(train$item_description)
        train$desclength[train$item_description == 'No description yet']<- NA
        cor(train$price,train$desc_length,use='complete.obs')
   
# 以下为部分文本分析内容，等待学习
corpus = Corpus(VectorSource(train$item_description))   #将要分析的变量加载到适当的格式中。
corpus = tm_map(corpus, tolower)          # 小写所有单词
corpus = tm_map(corpus, removePunctuation)  # 删除标点符号
corpus = tm_map(corpus, removeWords, stopwords("english"))  #去停用词
dataframe <- data.frame(text=sapply(corpus, identity),stringsAsFactors=F)    #转换为数据框
train$item_description = dataframe$text    #附加到原数据中