文本预处理
1.设置路径
setwd("e://r语言学习//r代码")
2.加载词典
installDict("D:\\R\\sources\\Dictionaries\\news.scel",dictname = "news1")
installDict("D:\\R\\sources\\Dictionaries\\news2.scel",dictname = "news2")
listDict()
3.加载文档
data <-readLines("d:\\R\\RWorkspace\\fhnews.txt",encoding ="UTF-8")
4.去除特殊词
dataTemp <- gsub("[0-9０１２３４５６７８９ < > ~]","",data)
5.分词
dataTemp <- segmentCN(dataTemp)
dataTemp[1:2]
6.去除停用词
stopwords<- unlist(read.table("D:\\R\\RWorkspace\\StopWords.txt",stringsAsFactors=F))
stopwords[50:100]
removeStopWords <- function(x,stopwords) {
temp <- character(0)
index <- 1
xLen <- length(x)
while (index <= xLen) {
if (length(stopwords[stopwords==x[index]]) <1)
temp<- c(temp,x[index])
index <- index +1
}
temp
}
> dataTemp2 <-lapply(dataTemp,removeStopWords,stopwords)
> dataTemp2[1:2]
文本分类
通过词频的余弦相似度做文本分类
1.加载语料库
library("tm")
reuters =VCorpus(VectorSource(doc_CN))
reuters <- tm_map(reuters, stripWhitespace)
2.删除停用词
data_stw<- unlist (read.table("E:\\text mining\\stopword\\中文停用词.txt",stringsAsFactors=F))
#head(data_stw,n=10)
reuters=tm_map(reuters,removeWords,data_stw)
3.生成TF-IDF特征
control=list(removePunctuation=T,minDocFreq=5,wordLengths = c(1, Inf),weighting = weightTfIdf)
doc.tdm=TermDocumentMatrix(reuters,control)
length(doc.tdm$dimnames$Terms)
tdm_removed=removeSparseTerms(doc.tdm, 0.97)
length(tdm_removed$dimnames$Terms)
mat = as.matrix(tdm_removed)####转换成文档矩阵
classifier = naiveBayes(mat[1:x,], as.factor(data$标题[1:x]) )##贝叶斯分类器，训练
predicted = predict(classifier, mat[z:y,]);#预测
A=table(data$标题[z:y], predicted)#预测交叉矩阵
predicted财经 禅道 军事 科技
财经 10 28 34 1
禅道 0 41 4 0
军事 0 10 25 0
科技 4 21 18 11
b1=length(which(predicted==data$标题[z:y]))/length(predicted)#计算召回率
b1[1] 0.4202899
补充：其它机器学习分类算法
library(RTextTools)
container = create_container(mat[1:y,], as.factor(data$标题[1:y]) ,
trainSize=1:x, testSize=1:y,virgin=TRUE)
models = train_models(container, algorithms=c("BAGGING" , "MAXENT" , "NNET" , "RF" , "SVM" , "TREE" ))
results = classify_models(container, models)
#How about the accuracy?
# recall accuracy
森林=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"FORESTS_LABEL"])
最大熵=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"MAXENTROPY_LABEL"])
决策树=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"TREE_LABEL"])
袋袋=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"BAGGING_LABEL"])
向量机=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"SVM_LABEL"])
神经网络=recall_accuracy(as.numeric(as.factor(data$标题[z:y])), results[,"NNETWORK_LABEL"])
a=c()
c=c()
e=c()
a=cbind( 随机森林=as.vector(results[,"FORESTS_LABEL"]), 决策树=as.vector(results[,"TREE_LABEL"]) , 支持向量机=as.vector(results[,"SVM_LABEL"]),贝叶斯=as.vector(predicted), 最大熵=as.vector(results[,"MAXENTROPY_LABEL"]),袋袋=as.vector(results[,"BAGGING_LABEL"]),神经网络=as.vector( results[,"NNETWORK_LABEL"]))
for(i in 1:length(results[,"FORESTS_LABEL"][z:y]))
{
b=table(a[i,])
c[i]<-names(which(b==max(table(a[i,]))))
}
模型预测=cbind(a,组合模型=c)
A=table(data$标题[z:y],c)
b=length(which(c==data$标题[z:y]))/length(c)
组合模型=b
e=c(贝叶斯=b1,森林=森林,最大熵=最大熵,决策树=决策树,袋袋=袋袋,向量机=向量机,神经网络=神经网络,组合投票=组合模型)
##结果该满意了吧！！！
e 贝叶斯 森林 最大熵 决策树 袋袋 向量机 神经网络 组合投票
0.4202899 1.0000000 1.0000000 0.5893720 1.0000000 0.3526570 0.9033816 1.0000000
文本聚类
文本聚类就没什么技术含量了，主要原因是其实非监督学习，效果一般不是很好。
data=t(mat[,1:50])
data.scale <- scale(data)
d <- dist(data.scale, method = "euclidean")
fit <- hclust(d, method="ward.D")
plot(fit,main="文本聚类")