MapReduce小示例

方便测试，特放此处

版本：1.0.4

使用新的api

package test;


import java.io.IOException;
import java.util.Iterator;
import java.util.StringTokenizer;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class WordCount {
	// mapper实现
	public static class WordCountMapper extends
			Mapper<LongWritable, Text, Text, IntWritable> {
		// 计数器
		private final static IntWritable one = new IntWritable(1);
		// key输出
		private Text word = new Text();


		protected void map(LongWritable key, Text value, Context context)
				throws java.io.IOException, InterruptedException {
			String line = value.toString();
			StringTokenizer st = new StringTokenizer(line);


			while (st.hasMoreTokens()) {
				word.set(st.nextToken());
				context.write(word, one);
			}
		};
	}


	// reduce实现
	public static class WordCountReduce extends
			Reducer<Text, IntWritable, Text, IntWritable> {
		@Override
		protected void reduce(Text key, java.lang.Iterable<IntWritable> arg1,
				Context context) throws IOException, InterruptedException {
			Iterator<IntWritable> iterator = arg1.iterator();
			int sum = 0;
			while (iterator.hasNext()) {
				sum += iterator.next().get();
			}
			context.write(key, new IntWritable(sum));
		};
	}


	public static void main(String[] args) throws Exception {
		// 创建一个job
		Configuration conf = new Configuration();
		Job job = new Job(conf, "WordCount");
		job.setJarByClass(WordCount.class);


		// 设置输入输出类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);


		// 设置map和reduce类
		job.setMapperClass(WordCountMapper.class);
		job.setReducerClass(WordCountReduce.class);


		// 设置输入输出流
		FileInputFormat.addInputPath(job, new Path("/tmp/a.txt"));
		FileOutputFormat.setOutputPath(job, new Path("/tmp/output"));


		job.submit();
	}


}