MapReduce(全局排序)

主要分类两次MapReduce, 最后一次MapReduce 的ReduceTask需要设置为1个

1. 自定义序列化数据类型

package com.gerry.bigdata.mapreduce.pagecountsort;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;;

public class PageCount implements WritableComparable { // 自定义数据类型,序列化

	private String page;
	private int count;

	public PageCount() {
		// TODO Auto-generated constructor stub
	}

	public void set(String page, int count) {
		this.page = page;
		this.count = count;
	}

	public String getPage() {
		return page;
	}

	public void setPage(String page) {
		this.page = page;
	}

	public int getCount() {
		return count;
	}

	public void setCount(int count) {
		this.count = count;
	}

	@Override
	public int compareTo(PageCount o) {
		// TODO Auto-generated method stub
		return o.getCount() - this.getCount() == 0 ? this.page.compareTo(o.getPage()) : o.getCount() - this.getCount();
	}

	@Override
	public void write(DataOutput out) throws IOException {
		// TODO Auto-generated method stub
		out.writeUTF(this.page);
		out.writeInt(this.count);

	}

	@Override
	public void readFields(DataInput in) throws IOException {
		// TODO Auto-generated method stub
		this.page = in.readUTF();
		this.count = in.readInt();

	}

	@Override
	public String toString() {
		// TODO Auto-generated method stub
		return this.page + "," + this.count;
	}

}

2. 第一次的MapReduce

package com.gerry.bigdata.mapreduce.pagecountsort;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class PageCountStep1 {

	public static class PageCountStep1Mapper extends Mapper {
		@Override
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String line = value.toString();
			String[] splits = line.split(" ");
			context.write(new Text(splits[1]), new IntWritable(1));

		}

	}

	public static class PageCountStep1Reducer extends Reducer {

		@Override
		protected void reduce(Text Key, Iterable values, Context context)
				throws IOException, InterruptedException {
			int count = 0;
			for (IntWritable value : values) {

				count += value.get();
			}

			context.write(Key, new IntWritable(count));

		}

	}

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);

		job.setJarByClass(PageCountStep1.class);
//		job.setJar("/home/gerry/pyspark/Bigdata/jars/pagecount.jar"); //在集群上运行
		job.setMapperClass(PageCountStep1Mapper.class);
		job.setReducerClass(PageCountStep1Reducer.class);

		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(IntWritable.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);

		Path inputPath = new Path("/home/gerry/pyspark/Bigdata/data/requests/input");
		Path outPath = new Path("/home/gerry/pyspark/Bigdata/data/requests/countout");
//		FileSystem fs = FileSystem.get(new URI("hdfs://172.16.0.2:9000/"), conf, "root");
//		if (fs.exists(outPath)) {
//			fs.delete(outPath, true);
//		}

		FileInputFormat.setInputPaths(job, inputPath);
		FileOutputFormat.setOutputPath(job, outPath);

		job.setNumReduceTasks(3);
		boolean result = job.waitForCompletion(true);
		System.exit(result ? 0 : 1);

	}

}

3. 第二次的MapReduce

package com.gerry.bigdata.mapreduce.pagecountsort;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class PageCountStep2 {
	
	public static class PageCountStep2Mapper extends Mapper{
		@Override
		protected void map(LongWritable key, Text value, Context context)
				throws IOException, InterruptedException {
			String line = value.toString();
			String[] splits = line.split("\t");
			PageCount pageCount = new PageCount();
			pageCount.set(splits[0], Integer.parseInt(splits[1]));
			context.write(pageCount, NullWritable.get());
		}
	}
	
	//借用mapreduce 的shuffle阶段的内部排序机制
	public static class PageCountStep2Reducer extends Reducer{
		
		@Override
		protected void reduce(PageCount key, Iterable values,Context context)
				throws IOException, InterruptedException {
			context.write(key, NullWritable.get());
		}
		
	}
	
	
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);

		job.setJarByClass(PageCountStep2.class);
//		job.setJar("/home/gerry/pyspark/Bigdata/jars/pagecount.jar"); //在集群上运行
		job.setMapperClass(PageCountStep2Mapper.class);
		job.setReducerClass(PageCountStep2Reducer.class);

		job.setMapOutputKeyClass(PageCount.class);
		job.setMapOutputValueClass(NullWritable.class);

		job.setOutputKeyClass(PageCount.class);
		job.setOutputValueClass(NullWritable.class);

		Path inputPath = new Path("/home/gerry/pyspark/Bigdata/data/requests/countout");
		Path outPath = new Path("/home/gerry/pyspark/Bigdata/data/requests/countoutput");
//		FileSystem fs = FileSystem.get(new URI("hdfs://172.16.0.2:9000/"), conf, "root");
//		if (fs.exists(outPath)) {
//			fs.delete(outPath, true);
//		}

		FileInputFormat.setInputPaths(job, inputPath);
		FileOutputFormat.setOutputPath(job, outPath);

		job.setNumReduceTasks(1);
		boolean result = job.waitForCompletion(true);
		System.exit(result ? 0 : 1);

	}
	

}

注意:本文涉及的代码都是在跨平台的Debug调试环境下,注意区分!!!