Data deduplication

题目:Data deduplication

描述

你的程序要求读入输入文件,在去掉所有数据中的重复数据后输出结果。在输入文件中每一行是一个元数据。

输入

输入是一组文本文件,在每个输入文件中每一行是一个数据。每一个元数据都是一个字符串。

输出文件

输出文件的每一行都是在输入文件中出现过的一个数据,并且输出文件中的每一行都不相同。

输入样例

input1:
2006-6-9 a
2006-6-10 b
2006-6-11 c
2006-6-12 d
2006-6-13 a
2006-6-14 b
2006-6-15 c
2006-6-11 c
input2:
2006-6-9 b
2006-6-10 a
2006-6-11 b
2006-6-12 d
2006-6-13 a
2006-6-14 c
2006-6-15 d
2006-6-11 c

输出样例:
2006-6-10 a 
2006-6-10 b 
2006-6-11 b 
2006-6-11 c 
2006-6-12 d 
2006-6-13 a 
2006-6-14 b 
2006-6-14 c 
2006-6-15 c 
2006-6-15 d 
2006-6-9 a 
2006-6-9 b

注意:
1 输出结果是按照字典顺序排序的;
2 每一行都是一个元数据;
3 重复数据在输出文件中也要输出一次。

整整干了一下午加晚上,悲催的。。。。

程序应该有漏洞,只是在单机下小数据文件可以测试通过。欢迎指正~~

代码如下:

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;


public class TextPair implements WritableComparable<TextPair> {

public Text first;
public Text second;

public TextPair(){
this.first=new Text();
this.second=new Text();
}

public TextPair(Text first, Text second) {
//super();
this.first = first;
this.second = second;
}
public TextPair(String first,String second){
this.first=new Text(first);
this.second=new Text(second);

}

public Text getFirst() {
return first;
}

public void setFirst(Text first) {
this.first = first;
}

public Text getSecond() {
return second;
}

public void setSecond(Text second) {
this.second = second;
}
public void set(Text first,Text second){
this.first=first;
this.second=second;
}

@Override
public int hashCode() {
// TODO Auto-generated method stub
return first.hashCode()*163+second.hashCode();
}

@Override
public boolean equals(Object obj) {
// TODO Auto-generated method stub
if(obj instanceof TextPair){
TextPair tp=(TextPair)obj;
return first.equals(tp.getFirst())&&second.equals(tp.getSecond());
}
return false;
}

@Override
public String toString() {
// TODO Auto-generated method stub
return first+"\t"+second;
}

@Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
first.readFields(in);
second.readFields(in);
}

@Override
public void write(DataOutput out) throws IOException {
// TODO Auto-generated method stub
first.write(out);
second.write(out);
}




@Override
public int compareTo(TextPair tp) {
// TODO Auto-generated method stub
int cmp=first.compareTo(tp.getFirst());
if(cmp!=0)
return cmp;
return second.compareTo(tp.getSecond());

}


}

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Partitioner;


public class FirstPartitioner extends Partitioner<TextPair, Text> {

@Override
public int getPartition(TextPair key, Text value, int numPartitions) {
// TODO Auto-generated method stub
return Math.abs(key.getFirst().hashCode()&Integer.MAX_VALUE)%numPartitions;
}

}

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;


public class GroupComparator extends WritableComparator {


public GroupComparator() {
super(TextPair.class, true);
// TODO Auto-generated constructor stub
}


@Override
public int compare(WritableComparable a, WritableComparable b) {
// TODO Auto-generated method stub
TextPair p1=(TextPair)a;
TextPair p2=(TextPair)b;
return p1.getFirst().compareTo(p2.getFirst());
}


}

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper;

public class DatadeduplicationMapper extends Mapper<LongWritable,Text,TextPair,Text>{

@Override
protected void map(LongWritable key, Text value,
org.apache.hadoop.mapreduce.Mapper.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
System.out.println("mapper");
String line=value.toString();
StringTokenizer token=new StringTokenizer(line);
String []str=new String[2];
int i=0;
TextPair tp=new TextPair();
while(token.hasMoreTokens()){
str[i]=token.nextToken();
i++;
}

tp.set(new Text(str[0]), new Text(str[1]));
System.out.println(tp);
System.out.println(str[1]);
context.write(tp, new Text(str[1]));

}


}

import java.awt.List;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.Vector;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class DatadeduplicationReducer extends
Reducer<TextPair, Text, Text, Text> {
@Override
protected void reduce(TextPair key, Iterable<Text> values,
Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
System.out.println("reducer");

Set set=new HashSet();
Vector<String>list=new Vector();

for(Text value:values){
//set.add(value.toString());
if(!list.contains(value.toString())){
list.add(value.toString());
}
System.out.println(key);
System.out.println(value);
}
for(int i=0;i<list.size();i++)
{
//System.out.println(it.next());
System.out.println((String)list.get(i));
context.write(key.getFirst(), new Text(list.get(i).toString()));
}
// context.write(key, values)
}

}

import java.io.IOException;



import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class Datadeduplication {

/**
*
@param args
*
@throws IOException
*
@throws ClassNotFoundException
*
@throws InterruptedException
*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
// TODO Auto-generated method stub
System.out.println("nihao");
Job job=new Job();
System.out.println("nihao");
job.setJobName("Datadeduplication");
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.setMapperClass(DatadeduplicationMapper.class);
job.setMapOutputKeyClass(TextPair.class);
job.setMapOutputValueClass(Text.class);


job.setPartitionerClass(FirstPartitioner.class);

job.setGroupingComparatorClass(GroupComparator.class);
//job.setSortComparatorClass(FirstGroupingComparator.class);
//job.setSortComparatorClass(GroupComparator.class);
//job.setGroupingComparatorClass(KeyComparator.class);

job.setReducerClass(DatadeduplicationReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);


job.waitForCompletion(false);

}

}



          



你可能感兴趣的:(Data)