hadoop中top-k问题解决

1.问题描述：在MapReduce中，想要输出最频繁出现的前k个单词。

                  问题输入：<单词，它出现的频率>

                  想要的输出：出现最多的前100个单词

   例如，输入是：

   hello  3

   word  4

   a   4

   moring  5

  goog  10

  bye  5

 (注意：中间的分割符是'\t')

  想要得到出现频率最多的前3个单词，则期望得到的结果为：

 goog  10

 moring  5

 bye  5

2.解决方案

   可以用一个map和一个reduce解决，map负责按频率降序输出键值对，把所有mapper的结果都输出到一个reduer中，reduce负责输出前3个出现频率最高的单词（这里输出是在reducer的cleanup（）函数中输出）

详情 参见http://www.cnblogs.com/hengli/archive/2012/12/04/2801619.html

 

3.程序代码

(1)

package sort;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.io.*;

public class wordSort {

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
		if(otherArgs.length != 2)
		{
			System.err.println("Usage: wordsort <in> <out>");
			System.exit(2);
		}
		Job job2 = new Job(conf,"word sort");    
		
	
		job2.setJarByClass(wordSort.class);
		job2.setMapperClass(SortMapper.class);
		job2.setReducerClass(SortReducer.class);
		
		job2.setMapOutputKeyClass(DesIntWritable.class);
		job2.setMapOutputValueClass(Text.class);
		//job2.setOutputKeyClass(Text.class);
		//job2.setOutputValueClass(DesIntWritable.class);
		job2.setNumReduceTasks(1);   //set the number of reducer = 1
		job2.setOutputKeyClass(NullWritable.class);
		job2.setOutputValueClass(Text.class);
		FileInputFormat.addInputPath(job2, new Path(otherArgs[0]));
		FileOutputFormat.setOutputPath(job2, new Path(otherArgs[1]));
		System.out.println("job2 start.....");
		job2.waitForCompletion(true);
		System.out.println("job2 done.");
		
	}

}

 (2)SortMapper类

   

package sort;

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Mapper.Context;

/**
 * 按键值降序输出
 * @author hx
 *
 */
public class SortMapper extends Mapper<LongWritable,Text,DesIntWritable,Text> {

	private DesIntWritable result = new DesIntWritable();
	private Text word = new Text();
	
	public void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException
	{
		
		String[] temp = value.toString().split("\t");

		if(temp != null && temp.length == 2)
		{
			result.set(Integer.parseInt(temp[1]));
			word.set(temp[0]);
			context.write(result, word);
		}
	}

}

 （3）SortReducer类

package sort;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.TreeMap;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Reducer.Context;

/**
 *  输出排在前3个到单词及其频数
 * @author hx
 *
 */
public class SortReducer extends Reducer<DesIntWritable, Text, NullWritable, Text> {
	
	public static final int k = 3;
	public List<Text> words = new ArrayList<Text>();
	
	public void reduce(DesIntWritable key,Iterable<Text> values,Context context) throws IOException, InterruptedException
	{
		for(Text val:values)
		{
			Text result = new Text();
			result.set(key + "\t" + val.toString());
			if(words.size() <= k-1)
				words.add(result);
		}
	}
	
	@Override
	protected void cleanup(Context context) throws IOException, InterruptedException
	{
		for(Text text :words)
		{
			context.write(NullWritable.get(), text);
		}
	}
	
}

 (4)DesIntWritable类

package sort;

import java.io.*;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/** A WritableComparable for ints. */
public class DesIntWritable implements WritableComparable {
  private int value;

  public DesIntWritable() {}

  public DesIntWritable(int value) { set(value); }

  /** Set the value of this DesIntWritable. */
  public void set(int value) { this.value = value; }

  /** Return the value of this DesIntWritable. */
  public int get() { return value; }

  public void readFields(DataInput in) throws IOException {
    value = in.readInt();
  }

  public void write(DataOutput out) throws IOException {
    out.writeInt(value);
  }

  /** Returns true iff <code>o</code> is a DesIntWritable with the same value. */
  public boolean equals(Object o) {
    if (!(o instanceof DesIntWritable))
      return false;
    DesIntWritable other = (DesIntWritable)o;
    return this.value == other.value;
  }

  public int hashCode() {
    return value;
  }

  /** Compares two DesIntWritables. */
  public int compareTo(Object o) {
    int thisValue = this.value;
    int thatValue = ((DesIntWritable)o).value;
    return (thisValue<thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
  }

  public String toString() {
    return Integer.toString(value);
  }

  /** A Comparator optimized for DesIntWritable. */ 
  public static class Comparator extends WritableComparator {
    public Comparator() {
      super(DesIntWritable.class);
    }

    public int compare(byte[] b1, int s1, int l1,
                       byte[] b2, int s2, int l2) {
      int thisValue = readInt(b1, s1);
      int thatValue = readInt(b2, s2);
      return (thisValue>thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
    }
  }

  static {                                        // register this comparator
    WritableComparator.define(DesIntWritable.class, new Comparator());
  }
}

 

参考：

[1] http://www.greenplum.com/blog/topics/hadoop/how-hadoop-mapreduce-can-transform-how-you-build-top-ten-lists

[2] http://www.cnblogs.com/hengli/archive/2012/12/04/2801619.html