Spark之WordCount

          配置好了Spark的集群环境，尝试弄个WordCount的例子，下面是具体的例子代码（javaAPI实现）

package com.test;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;

import scala.Tuple2;


public class WordCount {

	private static final Pattern SPACE = Pattern.compile(" ");
	public static void main(String[] args) {
		
		JavaSparkContext ctx = new JavaSparkContext("spark://192.168.100.103:7077","WordCount",System.getenv("SPARK_HOME"),JavaSparkContext.jarOfClass(WordCount.class));
		JavaRDD<String> lines = ctx.textFile("hdfs://192.168.100.103:9000/flume/test.log");
		ctx.addJar("/home/bms/WordCount.jar");
		
		JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String,String>(){

			@Override
			public Iterable<String> call(String arg0) throws Exception {
				
				return Arrays.asList(SPACE.split(arg0));
			}		
		});
		
		
		JavaPairRDD<String,Integer> ones = words.mapToPair(new PairFunction<String,String,Integer>(){

			@Override
			public Tuple2<String, Integer> call(String arg0) throws Exception {
								
				return new Tuple2<String,Integer>(arg0,1);
			}		
		});
		
		
		JavaPairRDD<String,Integer> counts = ones.reduceByKey(new Function2<Integer,Integer,Integer>(){

			@Override
			public Integer call(Integer arg0, Integer arg1) throws Exception {
				
				return arg0 + arg1;
			}		
		});
		
		List<Tuple2<String, Integer>> output = counts.collect();
		for(Tuple2<?,?> tuple : output){
			
		   System.out.println(tuple._1()+ ":" + tuple._2());
		}
		
		ctx.stop();
	}
}

注意：

1）ctx.addJar("/home/bms/WordCount.jar");是WordCount导出可运行的jar包的存放目录，我是直接在eclipse上直接编译运行的。在运行之前，先导出WordCount.jar。

2）spark://192.168.100.103:7077代表的Spark集群的master节点

3）SPARK_HOME代表的是Spark的环境变量

4）hdfs://192.168.100.103:9000/flume/test.log是我在hdfs上面存放的一个测试文件，文件内容是：

a a a

b b b

c c c

5）程序执行结果是：

b:3

a:3

c:3

6）WordCount初步试验成功。