使用Spark、Ansj分词进行词频统计

使用Spark进行词频统计
最近在做一个项目，要对大量的文本进行词频统计，发现常规的方法处理比较慢，所以尝试使用Spark进行计算。思路大致是这样：爬虫爬取京东的评论数据到mongodb,然后我从mongodb拉数据上传到HDFS,从HDFS拉数据然后用Spark进行词频统计。代码如下：

package com.aawant.respository.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.SQLContext;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;

import scala.Tuple2;

import com.aawant.mongo.repo.BaseMongoRepository;
import com.aawant.util.StringUtils;
import com.mongodb.BasicDBList;
import com.mongodb.CommandResult;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;

public class JingDongRespository {
    public static BaseMongoRepository base = new BaseMongoRepository("test");

    /*
     * 拉取京东数据
     */
    public static List getDistinctResult() {
        MongoClient m1 = new MongoClient("192.168.1.63:27017");
        @SuppressWarnings("deprecation")
        DB db = m1.getDB("test");
        DBCollection coll = db.getCollection("commodity");
        @SuppressWarnings("rawtypes")
        List list = coll.distinct("type");
        return list;
        // BasicDBList list = (BasicDBList) result.get("values");
//      for (int i = 0; i < list.size(); i++) {
//          System.out.println(list.get(i));
//      }
    }
    public static List listAllJdId(String typename) {
        List idList = new ArrayList();
        Query query = null;
        if (StringUtils.isNotEmpty(typename)) {
            query = Query.query(new Criteria().where("type").is(typename));
        }
        List