MapReduce—案例(四)简单数据去重

元数据
数据去重源数据:
2012-3-1 a
2012-3-2 b
2012-3-3 c
2012-3-4 d
2012-3-5 a
2012-3-6 b
2012-3-7 c
2012-3-3 c
2012-3-1 b
2012-3-2 a
2012-3-3 b
2012-3-4 d
2012-3-5 a
2012-3-6 c
2012-3-7 d
2012-3-3 c


最终结果:
2012-3-1 a
2012-3-1 b
2012-3-2 a
2012-3-2 b
2012-3-3 b
2012-3-3 c
2012-3-4 d
2012-3-5 a
2012-3-6 b
2012-3-6 c
2012-3-7 c
2012-3-7 d

思路:MapReduce的key可以自动去重,所以在reduce阶段,每次输出一个,就可以达到去重的目的。

package practice1;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
 * 数据去重
 * @author potter
 */
public class Practice3 {

	public static void main(String[] args) throws Exception {
		
		Configuration conf = new Configuration();
//		conf.set("fs.defaultFS", "hdfs:potter2:9000");//使用配置文件
//		System.setProperty("HADOOP_USER_NAME", "potter");//使用集群
		FileSystem fs = FileSystem.get(conf);//默认使用本地文件
		
		Job job = Job.getInstance();
		job.setJarByClass(Practice3.class);
		
		job.setMapperClass(Practice3Mapper.class);
		job.setReducerClass(Practice3Reducer.class);
		
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(NullWritable.class);
		
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);
		
//      String inputpath = args[0];//0代表的是输入第一个参数
//      String outpath = args[1]; //1代表的是输入的第二个参数
		
		Path input = new Path("D:\\practice\\input3\\work3.txt");
		Path output = new Path("D:\\practice\\input3\\output1");
		FileInputFormat.setInputPaths(job, input);
		FileOutputFormat.setOutputPath(job, output);
		
		if (fs.exists(output)) {
			fs.delete(output,true);
		}
		boolean isdone = job.waitForCompletion(true);
		
		System.exit(isdone ? 0 : 1);
		
	}
	
	public static class Practice3Mapper extends Mapper{
		//2012-3-1 a
		Text text = new Text();
		@Override
		protected void map(LongWritable key, Text value, Mapper.Context context)
				throws IOException, InterruptedException {
			
			String[] split = value.toString().trim().split(" ");
			String kk = split[0]+"\t"+split[1];
			text.set(kk);
			context.write(text, NullWritable.get());
		}
	}
	
	public static class Practice3Reducer extends Reducer{
		
		@Override
		protected void reduce(Text key, Iterable values, Context context)
				throws IOException, InterruptedException {
				context.write(key, NullWritable.get());
		}
	}
}

完成!!!

你可能感兴趣的:(MapReduce练习)