MapReduce查找隐藏好友

测试数据

tom rose
tom jim
tom smith
tom lucy
rose tom
rose lucy
rose smith
jim tom
jim lucy
smith jim
smith tom
smith rose

1自定义第一个Mapper：

public class UserMapper  extends Mapper<LongWritable, Text,Text, Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String line = value.toString();
        String[] sp = line.split(" ");
        context.write(new Text(sp[0]),new Text(sp[1]));
    }
}

2自定义第一个Reduce

public class UserReduce extends Reducer<Text, Text, Text, IntWritable> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        String name = key.toString();
        //由于迭代器只能迭代一次所以加个集合存储friend
        List<String> fs = new ArrayList<>();
        // 真实好友关系
        for (Text val : values) {
            String friend = val.toString();
            fs.add(friend);
            if (name.compareTo(friend) < 0){
                context.write(new Text(name + "-" + friend), new IntWritable(1));
            }
            else
            {
                context.write(new Text(friend + "-" + name), new IntWritable(1));
            }
        }
        // 推测好友关系
        // 假设好友之间相互认识
        for (int i = 0; i < fs.size() - 1; i++) {
            for (int j = i + 1; j < fs.size(); j++) {
                String f1 = fs.get(i);
                String f2 = fs.get(j);
                if(f1.compareTo(f2) < 0){
                    context.write(new Text(f1 + "-" + f2), new IntWritable(0));
                }
                else{
                    context.write(new Text(f2 + "-" + f1), new IntWritable(0));
                }
            }
        }
    }

3自定义第二个Mapper

public class MapperTwo extends Mapper<LongWritable, Text,Text,IntWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] arr = value.toString().split("\t");
        int n = Integer.parseInt(arr[1]);
        context.write(new Text(arr[0]), new IntWritable(n));
    }
}

4自定义第二个Reduce

public class ReducerTwo extends Reducer<Text,IntWritable,Text,NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        for (IntWritable val : values) {
            if (val.get() == 1){
                return;
            }
        }
        context.write(key, NullWritable.get());
    }
}

5自定义总Driver

public class UserDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 数据输入路径和输出路径
        args = new String[2];
        args[0] = "src/main/resources/input/test.txt";
        args[1] = "src/main/resources/output";
        // 读取配置文件
        Configuration cfg = new Configuration();
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFS", "file:///");
        // 新建一个任务
        Job job = Job.getInstance(cfg);
        // 设置主类
        job.setJarByClass(UserDriver.class);
        // Mapper
        job.setMapperClass(UserMapper.class);
        job.setReducerClass(UserReduce.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
       if( job.waitForCompletion(true) ){
           String []args1 = new String[2];
           args1[0] = "src/main/resources/output/part-r-00000";
           args1[1] = "src/main/resources/result";
           // 读取配置文件
           Configuration cfg1 = new Configuration();
           cfg1.set("mapreduce.framework.name", "local");
           cfg1.set("fs.defaultFS", "file:///");
           // 新建一个任务
           Job job2 = Job.getInstance(cfg);
           // 设置主类
           job2.setMapperClass(MapperTwo.class);
           job2.setReducerClass(ReducerTwo.class);
           job2.setMapOutputKeyClass(Text.class);
           job2.setMapOutputValueClass(IntWritable.class);
           job2.setOutputKeyClass(Text.class);
           job2.setOutputValueClass(IntWritable.class);
           FileInputFormat.addInputPath(job2, new Path(args1[0]));
           FileOutputFormat.setOutputPath(job2, new Path(args1[1]));
           boolean b = job2.waitForCompletion(true);
           System.exit(b?0:1);
       }
    }
}

效果图