Yarn核心——RPC(3) YarnRPC
hadoop 1.0中RPC的序列化机制是WritableRpcEngine，Yarn RPC采用ProtocolBuffer。
(1) 类型结构
(2) 定义RPC协议
package com.jackniu.yarnrpc.pb.api;
public interface Calculate {
int add(int num1, int num2);
int minus(int num1, int num2);
}
(3) 定义Proto文件
CalculateMessage.proto
option java_package ="com.jackniu.yarnrpc.pb.proto";
option java_outer_classname = "CalculateMessage";
option java_generic_services=true;
option java_generate_equals_and_hash=true;
message RequestProto{
required string methodName=1;
required int32 num1 =2;
required int32 num2=3;
}
message ResponseProto{
required int32 result =1;
}
CalculateServer.proto
option java_package ="com.jackniu.yarnrpc.pb.proto";
option java_outer_classname = "Calculate";
option java_generic_services=true;
option java_generate_equals_and_hash=true;
import "CalculateMessage.proto";
service CalculateService{
rpc add(RequestProto) returns (ResponseProto);
rpc minus(RequestProto) returns (ResponseProto);
}
生成对应的java文件 Calculate.java CalculateMessage.java(3) 定义接口协议和protobuf文件的关联
package com.jackniu.yarnrpc.pb.api;
import com.jackniu.yarnrpc.pb.proto.*;
import com.jackniu.yarnrpc.pb.proto.Calculate;
/**
* Created by JackNiu on 2017/9/1.
*/
public interface CalculatePB extends Calculate.CalculateService.BlockingInterface{
}
(4) 定义服务器端 对信息进行接受 解析
package com.jackniu.yarnrpc.pb.ipc;
import com.google.protobuf.BlockingService;
import com.google.protobuf.Descriptors;
import com.google.protobuf.Message;
import com.jackniu.yarnrpc.pb.proto.CalculateMessage;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
/**
* Created by JackNiu on 2017/9/2.
*/
public class Server extends Thread {
private Class protocol;
private BlockingService impl;
private int port;
private ServerSocket ss;
public Server(Class protocol, BlockingService impl,int port){
this.protocol = protocol;
this.impl = impl;
this.port = port;
}
@Override
public void run() {
System.out.println("accept... ");
Socket clientSocket = null;
DataOutputStream dos = null;
DataInputStream dis = null;
try {
ss = new ServerSocket(port);
} catch (IOException e) {
e.printStackTrace();
}
int testCount =10;
while(testCount -->0){
try{
clientSocket = ss.accept();
dos = new DataOutputStream(clientSocket.getOutputStream());
dis = new DataInputStream(clientSocket.getInputStream());
int dataLen = dis.readInt();
byte[] dataBuffer = new byte[dataLen];
int readCount = dis.read(dataBuffer);
byte[] result = processOneRpc(dataBuffer);
dos.writeInt(result.length);
dos.write(result);
dos.flush();
}catch(Exception e){
e.printStackTrace();
}
}
try{
dos.close();
dis.close();
ss.close();
}catch(Exception e){
};
}
public byte[] processOneRpc(byte[] data) throws Exception{
CalculateMessage.RequestProto request = CalculateMessage.RequestProto.parseFrom(data);
String methodName = request.getMethodName();
Descriptors.MethodDescriptor methodDescriptor = impl.getDescriptorForType().findMethodByName(methodName);
Message response = impl.callBlockingMethod(methodDescriptor,null,request);
return response.toByteArray();
}
}
(5) 定义服务端服务Service，用以启动后台服务，并启动特定的事件服务，这里就是Server(Calculate的)
package com.jackniu.yarnrpc.pb.server.business;
import com.google.protobuf.BlockingService;
import com.jackniu.yarnrpc.pb.api.Calculate;
import com.jackniu.yarnrpc.pb.ipc.Server;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
/**
* Created by JackNiu on 2017/9/2.
*/
public class CalculateService implements Calculate {
private Server server = null;
private final Class protocol = Calculate.class;
private final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
private final String protoPackage ="com.jackniu.yarnrpc.pb.proto";
private final String host = "localhost";
private final int port = 8888;
public CalculateService(){
}
public int add(int num1, int num2) {
return num1+num2;
}
public int minus(int num1, int num2) {
return num1-num2;
}
// return ： org.jackniu.yarnrpc.pb.api.CalculatePBServiceImpl
public Class getPbServiceImplClass(){
String packageName = protocol.getPackage().getName();
String className = protocol.getSimpleName();
String pbServiceImplName = packageName + "." + className + "PBServiceImpl";
Class clazz = null;
try{
clazz = Class.forName(pbServiceImplName, true, classLoader);
}catch(ClassNotFoundException e){
System.err.println(e.toString());
}
System.out.println(clazz);
return clazz;
}
public Class getProtoClass(){
String className = protocol.getSimpleName();
String protoClazzName = protoPackage + "." + className + "$" + className + "Service";
Class clazz = null;
try{
clazz = Class.forName(protoClazzName, true, classLoader);
}catch(ClassNotFoundException e){
System.err.println(e.toString());
}
System.out.println(clazz);
return clazz;
}
public void createServer() {
Class pbServiceImpl = getPbServiceImplClass();
Constructor constructor = null;
try {
constructor = pbServiceImpl.getConstructor(protocol);
constructor.setAccessible(true);
} catch (NoSuchMethodException e) {
System.err.print(e.toString());
}
Object service = null; // instance of CalculatePBServiceImpl
try {
service = constructor.newInstance(this);
} catch (InstantiationException e) {
} catch (IllegalArgumentException e) {
} catch (IllegalAccessException e) {
} catch (InvocationTargetException e) {
}
Class pbProtocol = service.getClass().getInterfaces()[0];
System.out.println(pbProtocol);
Class protoClazz = getProtoClass();
System.out.println(protoClazz);
Method method = null;
try {
method = protoClazz.getMethod("newReflectiveBlockingService", pbProtocol.getInterfaces()[0]);
} catch (Exception e) {
e.printStackTrace();
}
try {
createServer(pbProtocol,(BlockingService) method.invoke(null,service));
}catch (Exception e){
e.printStackTrace();
}
}
public void createServer(Class pbProtocol,BlockingService service){
server= new Server(pbProtocol,service,port);
server.start();
}
public void init(){
createServer();
}
public static void main(String[] args) {
CalculateService cs= new CalculateService();
cs.init();
}
}
(6) 接口协议实现，实现的是Proto定义的方法
package com.jackniu.yarnrpc.pb.api;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import com.jackniu.yarnrpc.pb.proto.CalculateMessage;
/**
* Created by JackNiu on 2017/9/2.
*/
public class CalculatePBServiceImpl implements CalculatePB {
public Calculate real;
public CalculatePBServiceImpl(Calculate impl){
this.real = impl;
}
public CalculateMessage.ResponseProto add(RpcController controller, CalculateMessage.RequestProto request) throws ServiceException {
CalculateMessage.ResponseProto proto = CalculateMessage.ResponseProto.getDefaultInstance();
CalculateMessage.ResponseProto.Builder build = CalculateMessage.ResponseProto.newBuilder();
int add1 = request.getNum1();
int add2 = request.getNum2();
int sum = real.add(add1, add2);
CalculateMessage.ResponseProto result = null;
build.setResult(sum);
result = build.build();
return result;
}
public CalculateMessage.ResponseProto minus(RpcController controller, CalculateMessage.RequestProto request) throws ServiceException {
CalculateMessage.ResponseProto proto = CalculateMessage.ResponseProto.getDefaultInstance();
CalculateMessage.ResponseProto.Builder build = CalculateMessage.ResponseProto.newBuilder();
int add1 = request.getNum1();
int add2 = request.getNum2();
int sum = real.minus(add1, add2);
CalculateMessage.ResponseProto result = null;
build.setResult(sum);
result = build.build();
return result;
}
}
(7) 客户端实现
package com.jackniu.yarnrpc.pb.client;
import com.jackniu.yarnrpc.pb.api.Calculate;
import com.jackniu.yarnrpc.pb.proto.CalculateMessage;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.net.Socket;
import java.util.Random;
/**
* Created by JackNiu on 2017/9/2.
*/
public class CalculateClient implements Calculate{
public int add(int num1, int num2) {
return socketoperation("add",num1,num2);
}
public int minus(int num1, int num2) {
return socketoperation("minus",num1,num2);
}
public int socketoperation(String op,int num1,int num2){
Socket s = null;
DataOutputStream out = null;
DataInputStream in = null;
int ret = 0;
try{
s= new Socket("localhost", 8888);
out = new DataOutputStream(s.getOutputStream());
in = new DataInputStream(s.getInputStream());
CalculateMessage.RequestProto.Builder builder = CalculateMessage.RequestProto.newBuilder();
builder.setMethodName(op);
builder.setNum1(num1);
builder.setNum2(num2);
CalculateMessage.RequestProto request = builder.build();
byte [] bytes = request.toByteArray();
out.writeInt(bytes.length);
out.write(bytes);
out.flush();
int dataLen = in.readInt();
byte[] data = new byte[dataLen];
int count = in.read(data);
if(count != dataLen){
System.err.println("something bad happened!");
}
CalculateMessage.ResponseProto result = CalculateMessage.ResponseProto.parseFrom(data);
System.out.println(num1 + " " + op + " " + num2 + "=" + result.getResult());
ret = result.getResult();
}catch(Exception e){
e.printStackTrace();
}finally {
try{
in.close();
out.close();
s.close();
}catch(Exception e){
e.printStackTrace();
}
}
return ret;
}
public static void main(String[] args) {
CalculateClient client = new CalculateClient();
int testCount = 5;
Random rand = new Random();
while(testCount-- > 0){
int a = rand.nextInt(100);
int b = rand.nextInt(100);
client.add(a,b);
client.minus(a, b);
}
}
}
(8) 控制台输出
Server:
class com.jackniu.yarnrpc.pb.api.CalculatePBServiceImpl
interface com.jackniu.yarnrpc.pb.api.CalculatePB
class com.jackniu.yarnrpc.pb.proto.Calculate$CalculateService
class com.jackniu.yarnrpc.pb.proto.Calculate$CalculateService
accept...
-----------------------------------------------
client:
90 add 96=186
90 minus 96=-6
75 add 49=124
75 minus 49=26
7 add 81=88
7 minus 81=-74
41 add 93=134
41 minus 93=-52
76 add 88=164
整个YarnRpc的简单实现如上，还有很多丰富的地方，网络通信现在应该是Netty的Nio通信，原理还是很简单的，可以增加很多别的特性，看个人需求。 理解还是有点麻烦。
差不多能够理解Yarn 中的基于Protobuf的通信方式，源码理解也希望有帮助。
参考网址: http://standalone.iteye.com/blog/1727544