MapReduce编程小案例.7th—求用户之间的共同好友及有哪些好友
MapReduce编程小案例.7th—求用户之间的共同好友及有哪些好友
需求：有如下一组数据
哪些用户两两之间有共同好友，及共同好友都是哪些人
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J需要得到如下数据：
A-B C,E
A-C D,F
...实现代码：
分2步：
1.CommonFriendsOne类实现
package cn.edu360.mr.friends;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class CommonFriendsOne {
public static class CommonFriendsOneMapper extends Mapper{
Text k = new Text();
Text v = new Text();
//A: B,C,D,E,F,D
//输出：B - A ,C - A ,D - A ...
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] userAndFriends = value.toString().split(":");
String user = userAndFriends[0];
String[] friends = userAndFriends[1].split(",");
v.set(user);
for (String f : friends) {
k.set(f);
context.write(k, v);
}
}
}
public static class CommonFriendsOneReducer extends Reducer{
@Override
protected void reduce(Text friend, Iterable users, Reducer.Context context)
throws IOException, InterruptedException {
ArrayList userList = new ArrayList();
for (Text user : users) {
userList.add(user.toString());
}
Collections.sort(userList);
for (int i =0 ; i < userList.size() - 1; i++) {
for(int j = i+1 ; j < userList.size(); j++) {
context.write(new Text(userList.get(i) + "-" + userList.get(j)), friend);
}
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(CommonFriendsOne.class);
job.setMapperClass(CommonFriendsOneMapper.class);
job.setReducerClass(CommonFriendsOneReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path("F:\\mrdata\\friend\\input"));
FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\friend\\out1"));
job.waitForCompletion(true);
}
}
2.CommonFriendsTwo类实现
package cn.edu360.mr.friends;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class CommonFriendsTwo {
public static class CommonFriendsTwoMapper extends Mapper{
Text k = new Text();
Text v = new Text();
@Override
protected void map(LongWritable key, Text value, Mapper.Context context)
throws IOException, InterruptedException {
String[] split = value.toString().split("\t");
k.set(split[0]);
v.set(split[1]);
context.write(k, v);
}
}
public static class CommonFriendsTwoReducer extends Reducer{
Text v = new Text();
@Override
protected void reduce(Text friendGroup, Iterable commonFriend, Reducer.Context context)
throws IOException, InterruptedException {
StringBuilder sb = new StringBuilder();
for (Text cf : commonFriend) {
sb.append(cf.toString()).append(" ");
}
v.set(sb.toString());
context.write(friendGroup, v);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(CommonFriendsOne.class);
job.setMapperClass(CommonFriendsTwoMapper.class);
job.setReducerClass(CommonFriendsTwoReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path("F:\\mrdata\\friend\\out1"));
FileOutputFormat.setOutputPath(job, new Path("F:\\mrdata\\friend\\out2"));
job.waitForCompletion(true);
}
}