MapReduce 新旧WordCount 代码解读

一、mapreduce 的 wordcount 旧API

import java.io.IOException;  

import java.util.Iterator;  

import java.util.StringTokenizer;  

import org.apache.hadoop.fs.Path;  

import org.apache.hadoop.io.IntWritable;  

import org.apache.hadoop.io.LongWritable;  

import org.apache.hadoop.io.Text;  

import org.apache.hadoop.mapred.FileInputFormat;  

import org.apache.hadoop.mapred.FileOutputFormat;  

import org.apache.hadoop.mapred.JobClient;  

import org.apache.hadoop.mapred.JobConf;  

import org.apache.hadoop.mapred.MapReduceBase;  

import org.apache.hadoop.mapred.Mapper;  

import org.apache.hadoop.mapred.OutputCollector;  

import org.apache.hadoop.mapred.Reducer;  

import org.apache.hadoop.mapred.Reporter;  

import org.apache.hadoop.mapred.TextInputFormat;  

import org.apache.hadoop.mapred.TextOutputFormat;  

/** 

 *  

 * 描述：WordCount explains by Felix 

 * @author Hadoop Dev Group 

 */  

public class WordCount  

{  

    /**  

     * MapReduceBase类:实现了Mapper和Reducer接口的基类（其中的方法只是实现接口，而未作任何事情） 

     * Mapper接口： 

     * WritableComparable接口：实现WritableComparable的类可以相互比较。所有被用作key的类应该实现此接口。 

     * Reporter 则可用于报告整个应用的运行进度，本例中未使用。  

     *  

     */  

    public static class Map extends MapReduceBase implements  

            Mapper<LongWritable, Text, Text, IntWritable>  

    {  

        /** 

         * LongWritable, IntWritable, Text 均是 Hadoop 中实现的用于封装 Java 数据类型的类，这些类实现了WritableComparable接口， 

         * 都能够被串行化从而便于在分布式环境中进行数据交换，你可以将它们分别视为long,int,String 的替代品。 

         */  

        private final static IntWritable one = new IntWritable(1);  

        private Text word = new Text();  

          

        /** 

         * Mapper接口中的map方法： 

         * void map(K1 key, V1 value, OutputCollector<K2,V2> output, Reporter reporter) 

         * 映射一个单个的输入k/v对到一个中间的k/v对 

         * 输出对不需要和输入对是相同的类型，输入对可以映射到0个或多个输出对。 

         * OutputCollector接口：收集Mapper和Reducer输出的<k,v>对。 

         * OutputCollector接口的collect(k, v)方法:增加一个(k,v)对到output 

         */  

        public void map(LongWritable key, Text value,  

                OutputCollector<Text, IntWritable> output, Reporter reporter)  

                throws IOException  

        {  

            String line = value.toString();  

            StringTokenizer tokenizer = new StringTokenizer(line);  

            while (tokenizer.hasMoreTokens())  

            {  

                word.set(tokenizer.nextToken());  

                output.collect(word, one);  

            }  

        }  

    }  

    public static class Reduce extends MapReduceBase implements  

            Reducer<Text, IntWritable, Text, IntWritable>  

    {  

        public void reduce(Text key, Iterator<IntWritable> values,  

                OutputCollector<Text, IntWritable> output, Reporter reporter)  

                throws IOException  

        {  

            int sum = 0;  

            while (values.hasNext())  

            {  

                sum += values.next().get();  

            }  

            output.collect(key, new IntWritable(sum));  

        }  

    }  

    public static void main(String[] args) throws Exception  

    {  

        /** 

         * JobConf：map/reduce的job配置类，向hadoop框架描述map-reduce执行的工作 

         * 构造方法：JobConf()、JobConf(Class exampleClass)、JobConf(Configuration conf)等 

         */  

        JobConf conf = new JobConf(WordCount.class);  

        conf.setJobName("wordcount");           //设置一个用户定义的job名称  

        conf.setOutputKeyClass(Text.class);    //为job的输出数据设置Key类  

        conf.setOutputValueClass(IntWritable.class);   //为job输出设置value类  

        conf.setMapperClass(Map.class);         //为job设置Mapper类  

        conf.setCombinerClass(Reduce.class);      //为job设置Combiner类  

        conf.setReducerClass(Reduce.class);        //为job设置Reduce类  

        conf.setInputFormat(TextInputFormat.class);    //为map-reduce任务设置InputFormat实现类  

        conf.setOutputFormat(TextOutputFormat.class);  //为map-reduce任务设置OutputFormat实现类  

        /** 

         * InputFormat描述map-reduce中对job的输入定义 

         * setInputPaths():为map-reduce job设置路径数组作为输入列表 

         * setInputPath()：为map-reduce job设置路径数组作为输出列表 

         */  

        FileInputFormat.setInputPaths(conf, new Path(args[0]));  

        FileOutputFormat.setOutputPath(conf, new Path(args[1]));  

        JobClient.runJob(conf);         //运行一个job  

    }  

} 

二、mapreduce 的 wordcount 新API

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {

  /**
   * TokenizerMapper 继续自 Mapper<Object, Text, Text, IntWritable>
   *
   * [一个文件就一个map,两个文件就会有两个map]
   * map[这里读入输入文件内容 以" \t\n\r\f" 进行分割，然后设置 word ==> one 的key/value对]
   *
   * @param Object  Input key Type:
   * @param Text    Input value Type:
   * @param Text    Output key Type:
   * @param IntWritable Output value Type:
   *
   * Writable的主要特点是它使得Hadoop框架知道对一个Writable类型的对象怎样进行serialize以及deserialize.
   * WritableComparable在Writable的基础上增加了compareT接口，使得Hadoop框架知道怎样对WritableComparable类型的对象进行排序。
   *
   * @author yangchunlong.tw
   *
   */
  public static class TokenizerMapper
       extends Mapper<Object, Text, Text, IntWritable>{

    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();
    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
      StringTokenizer itr = new StringTokenizer(value.toString());
      while (itr.hasMoreTokens()) {
        word.set(itr.nextToken());
        context.write(word, one);
      }
    }
  }

  /**
   * IntSumReducer 继承自 Reducer<Text,IntWritable,Text,IntWritable>
   *
   * [不管几个Map,都只有一个Reduce,这是一个汇总]
   * reduce[循环所有的map值,把word ==> one 的key/value对进行汇总]
   *
   * 这里的key为Mapper设置的word[每一个key/value都会有一次reduce]
   *
   * 当循环结束后，最后的确context就是最后的结果.
   *
   * @author yangchunlong.tw
   *
   */
  public static class IntSumReducer
       extends Reducer<Text,IntWritable,Text,IntWritable> {
    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context
                       ) throws IOException, InterruptedException {
      int sum = 0;
      for (IntWritable val : values) {
        sum += val.get();
      }
      result.set(sum);
      context.write(key, result);
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    /**
     * 这里必须有输入/输出
     */
    if (otherArgs.length != 2) {
      System.err.println("Usage: wordcount <in> <out>");
      System.exit(2);
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(WordCount.class);//主类
    job.setMapperClass(TokenizerMapper.class);//mapper
    job.setCombinerClass(IntSumReducer.class);//作业合成类
    job.setReducerClass(IntSumReducer.class);//reducer
    job.setOutputKeyClass(Text.class);//设置作业输出数据的关键类
    job.setOutputValueClass(IntWritable.class);//设置作业输出值类
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));//文件输入
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));//文件输出
    System.exit(job.waitForCompletion(true) ? 0 : 1);//等待完成退出.
  }
}