MapReduce列子WordCount

1. 写程序MyWordCount.java

   package org.myorg;
   
   import java.io.IOException;
   import java.util.*;
   
   import org.apache.hadoop.fs.Path;
   import org.apache.hadoop.conf.*;
   import org.apache.hadoop.io.*;
   import org.apache.hadoop.mapred.*;
   import org.apache.hadoop.util.*;
   
   public class WordCount {
   	public static void main(String[] args) throws Exception {
   		JobConf conf = new JobConf(WordCount.class);
   		conf.setJobName("wordcount");
   
   		//conf.setNumReduceTasks(0);		
   		
   		conf.setOutputKeyClass(Text.class);
   		conf.setOutputValueClass(IntWritable.class);
   		
   		conf.setMapperClass(Map.class);
   		conf.setCombinerClass(Reduce.class);
   		//conf.setReducerClass(Reduce.class);
   	
   		conf.setInputFormat(TextInputFormat.class);
   		conf.setOutputFormat(TextOutputFormat.class);
   		
   		FileInputFormat.setInputPaths(conf, new Path(args[0]));
   		FileOutputFormat.setOutputPath(conf, new Path(args[1]));
   		
   		JobClient.runJob(conf);
   	}
   
   	public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
   		private final static IntWritable one = new IntWritable(1);
   		private Text word = new Text();
   
   		public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
   			String line = value.toString();
   			StringTokenizer tokenizer = new StringTokenizer(line);
   			while (tokenizer.hasMoreTokens()) {
   				word.set(tokenizer.nextToken());
   				output.collect(word, one);
   			}
   		}
   	}
   
   	public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
   		public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
   			int sum = 0;
   			while (values.hasNext()) {
   				sum += values.next().get();
   			}
   			output.collect(key, new IntWritable(sum));
   		}
   	}
   }
   

    源文档 <http://hadoop.apache.org/docs/r0.18.3/mapred_tutorial.html>

   注：conf.setInputFormat(TextInputFormat.class);    //TextInputFormat是默认的InputFormat。这说明Map类的键是LongWritable类型，存储整个文件的字节偏移量，值是Text类型，是一行内容。StringTokenizer 把行按空格拆分成单词。
   conf.setOutputFormat(TextOutputFormat.class);     //输出格式为TextOutputFormat，把输出记录写成文本行。键值可以使任何类型，因为可以用toString()方法转成字符串。这里的输出键是Text类型，值是IntWritable类型。

     2. 编译

         Mkdir  wordcountsource

        hduser@ubuntu:/usr/local/hadoop$ javac -classpath hadoop-core-1.1.1.jar -d wordcountsource                       MyWordCount.java
        编译java到wordcountsource文件夹下
          如果出现错误：error while writing Map: could not create parent directories
         说明没有写入input文件的权限

     3.生成jar

        hduser@ubuntu:/usr/local/hadoop$ sudo jar -cvf MyWordCount.jar  -C wordcountsource/ .

       在当前目录下生成MyWordCount.jar

      4.在input文件夹下创建file0和file1

         hduser@ubuntu:/usr/local/hadoop/input$ mkdir input    //input文件夹为输入

         hduser@ubuntu:/usr/local/hadoop/input$ sudo gedit file0
         hduser@ubuntu:/usr/local/hadoop/input$ sudo gedit file1

      5.运行MyWordCount.jar

       hduser@ubuntu:/usr/local/hadoop$ sudo bin/hadoop jar MyWordCount.jar org.myorg.MyWordCount              input output

       确保有创建output的权限

      6.查看结果

      hduser@ubuntu:/usr/local/hadoop$ cat output/part-00000

      Bye 1
     Hello 1
    World 2

转载