MapReduce的API操作

获取每一个学生的最高分 — score2.txt

**Mapper类**
	public class MaxScoreMapper extends Mapper {
		public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String line = value.toString();
			// 以空格为符号进行切分
			String[] arr = line.split(" ");
			Text name = new Text(arr[0]);
			IntWritable score = new IntWritable(Integer.parseInt(arr[1]));
			context.write(name, score);	
		}
	}
**Reduce类**
		public class MaxScoreReducer extends Reducer {
		public void reduce(Text _key, Iterable values, Context context) throws IOException, InterruptedException {
			int max = 0;
			for (IntWritable val : values) {
				if(max < val.get())
					max = val.get();
			}		
			context.write(_key, new IntWritable(max));
		}
	}
**Driver类**
	public class MaxScoreDriver {
		public static void main(String[] args) throws Exception {
			Configuration conf = new Configuration();
			Job job = Job.getInstance(conf, "JobName");
			job.setJarByClass(cn.tedu.maxscore.MaxScoreDriver.class);
			job.setMapperClass(MaxScoreMapper.class);
			job.setReducerClass(MaxScoreReducer.class);
			job.setOutputKeyClass(Text.class);
			job.setOutputValueClass(IntWritable.class);
			FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.60.132:9000/mr/score2.txt"));
			FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.60.132:9000/msresult"));
			if (!job.waitForCompletion(true))
				return;
		}
	}

序列化/反序列化机制

当自定义一个类之后，如果想要产生的对象在hadoop中进行传输，那么需要这个类实现Writable的接口进行序列化/反序列化

案例:统计每一个人产生的总流量

## 流量类

public class Flow implements Writable{

	private String phone;
	private String city;
	private String name;
	private int flow;

	public String getPhone() {
		return phone;
	}

	public void setPhone(String phone) {
		this.phone = phone;
	}

	public String getCity() {
		return city;
	}

	public void setCity(String city) {
		this.city = city;
	}

	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public int getFlow() {
		return flow;
	}

	public void setFlow(int flow) {
		this.flow = flow;
	}

	// 反序列化
	@Override
	public void readFields(DataInput in) throws IOException {
		// 按照序列化的顺序一个一个将数据读取出来
		this.phone = in.readUTF();
		this.city = in.readUTF();
		this.name = in.readUTF();
		this.flow = in.readInt();
	}

	// 序列化
	@Override
	public void write(DataOutput out) throws IOException {
		// 按照顺序将属性一个一个的写出即可
		out.writeUTF(phone);
		out.writeUTF(city);
		out.writeUTF(name);
		out.writeInt(flow);
	}
}
## Mapper类
public class FlowMapper extends Mapper {

	public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
		String line = value.toString();
		String[] arr = line.split(" ");
		Flow f = new Flow();
		f.setPhone(arr[0]);
		f.setCity(arr[1]);
		f.setName(arr[2]);
		f.setFlow(Integer.parseInt(arr[3]));
		context.write(new Text(f.getPhone()), f);
	}
}

## Reduce类
public class FlowReducer extends Reducer {

	public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
		
		int sum = 0;
		String name = null;
		for (Flow val : values) {
			name = val.getName();
			sum += val.getFlow();
		}
		
		context.write(new Text(key.toString() + " " + name), new IntWritable(sum));
	}

}

## Driver类
public class FlowDriver {

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf, "JobName");
		job.setJarByClass(cn.tedu.flow.FlowDriver.class);
		job.setMapperClass(FlowMapper.class);
		job.setReducerClass(FlowReducer.class);

		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(Flow.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);

		FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.60.132:9000/mr/flow.txt"));
		FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.60.132:9000/flowresult"));

		if (!job.waitForCompletion(true))
			return;
	}

}

**

注意

**:需要被序列化的类中，进行序列化的write和反序列化的read的属性要一一对应，否则在Reduce进行反序列化的时候属性的值可能会颠倒