HADOOP实现查找QQ共同好友功能
HADOOP实现查找QQ共同好友功能
需求：
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J
求出哪些人互相之间是好友，及他们的共同好友都是谁
比如:
A-B : C,E
A-C : D,F
map中逻辑如下：
/**
* 入参：
* A:B,C,D,F,E,O
* B:A,C,E,K
* ....
* ....
* A出参：
* A-B:C,D,F,E,O
* A-C:B,D,F,E,O
* .....
* B出参：
* A-B:C,E,K
* B-C:A,E,K
* ...
* 分析：
* 1，要找到互为好友的两个人之间的好友是谁，首先找出A的好友有谁，以及除它之外还有谁
*,2，要使互为好友的两个人在maptask中处理完成之后汇聚到reducetask，就必须使key相同，可以根据hashcode来排顺序
*/
public class QQFriendMapper extends Mapper<LongWritable,Text,Text,Text> {
Text keyText = new Text();
Text valueText = new Text();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
String[] allFriends = line.split(":");
String userMain = allFriends[0];
String userOther = allFriends[1];
System.out.println("userMain======="+userMain);
System.out.println("userOther======="+userOther);
String[] friends = userOther.split(",");
List<String> friendsList = Arrays.asList(friends);
String keyString = "";
for (String friend : friends){
if (userMain.hashCode()<friend.hashCode()){//排列key的顺序
keyString = userMain +"-"+ friend +":";
}else {
keyString = friend +"-"+ userMain +":";
}
String valueString = "";
for (String valueStr : friendsList){
if (!valueStr.equals(friend)){
valueString += valueStr+",";
}
}
keyText.set(keyString);
valueText.set(valueString);
System.out.println("keyString=========="+keyString);
System.out.println("valueString=========="+valueString);
context.write(keyText,valueText);
}
}
}
reduce中逻辑如下：
/**
* 入参：
*
* .....
*
* 出参：
* A-B: C,E
* .....
* 分析：有的好友可能是单向好友(即迭代器中只有一组数据)，需要过滤掉
*/
public class QQFriendReduce extends Reducer<Text,Text,Text,Text> {
Text valueText = new Text();
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
System.out.println(key.toString());
Iterator<Text> iterator = values.iterator();
Text groupA = new Text();
Text groupB = new Text();
int i = 1;//过滤标识，互为好友的两个人，迭代器中只有两个，处理时，只处理i==3的数据
while (iterator.hasNext()){
if (i == 2){
groupB.set(iterator.next());
System.out.println("groupB====="+groupB);
}else if (i == 1){
groupA.set(iterator.next());
System.out.println("groupA========"+groupA);
}
i++;
}
System.out.println(groupA);
System.out.println(groupB);
System.out.println("i============"+i);
if (i == 3){
String[] groupStrsA = groupA.toString().split(",");
String[] groupStrsB = groupB.toString().split(",");
String equalFriend = "";
for (String strA : groupStrsA){
for (String strB : groupStrsB){
if (strA.equals(strB)){
equalFriend += strA + ",";
}
}
}
System.out.println(groupA);
System.out.println(groupB);
System.out.println("equalFriend========="+equalFriend);
if (!equalFriend.equals("") && equalFriend != null){
equalFriend = equalFriend.substring(0,equalFriend.length()-1);
}
valueText.set(equalFriend);
context.write(key,valueText);
}
}
}