[HBase]RPC框架之 region server实现

HBase的RPC沿用了hadoop的RPC部分代码。HMaster,RegionServer和client都是通过RPC交换数据的。其实大抵相似。RegionServer端RPC部分类图如下:

 

 HBaseServer核心类,实现了Reactor模型,主线程Listener负责accept外部连接,子线程Reader负责连接的具体读写操作,将数据反序列化成Call对象,通过Queue交给后面的Handler线程处理,Handler线程发起反射调用,并将response数据交给Responder线程处理,Responder线程将数据最终写回给client。

 

0.94代码实现如下:

HRegionServer启动时,构造函数中

//端口默认60020
 this.rpcServer = HBaseRPC.getServer(this,
      new Class[]{HRegionInterface.class, HBaseRPCErrorHandler.class,
        OnlineRegions.class},
        initialIsa.getHostName(), // BindAddress is IP we got for this server.
        initialIsa.getPort(),
        conf.getInt("hbase.regionserver.handler.count", 10),
        conf.getInt("hbase.regionserver.metahandler.count", 10),
        conf.getBoolean("hbase.rpc.verbose", false),
        conf, QOS_THRESHOLD);

 之后初始化HBaseServer,主要就是初始化Listener和Responder

protected HBaseServer(String bindAddress, int port,
                        Class paramClass, int handlerCount,
                        int priorityHandlerCount, Configuration conf, String serverName,
                        int highPriorityLevel)
    throws IOException {
    this.bindAddress = bindAddress;
    ......

    // temporary backward compatibility
    String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size");
    if (oldMaxQueueSize == null) {
      this.maxQueueLength =
        this.conf.getInt("ipc.server.max.callqueue.length",
          handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
    } else {
      LOG.warn("ipc.server.max.queue.size was renamed " +
               "ipc.server.max.callqueue.length, " +
               "please update your configuration");
      this.maxQueueLength = Integer.getInteger(oldMaxQueueSize);
    }

	//默认queue length是100,queue size是3K
    this.maxQueueSize =
      this.conf.getInt("ipc.server.max.callqueue.size",
        DEFAULT_MAX_CALLQUEUE_SIZE);
     this.readThreads = conf.getInt(
        "ipc.server.read.threadpool.size",
        10);
    this.callQueue  = new LinkedBlockingQueue(maxQueueLength);
    if (priorityHandlerCount > 0) {
      this.priorityCallQueue = new LinkedBlockingQueue(maxQueueLength); // TODO hack on size
    } else {
      this.priorityCallQueue = null;
    }
    this.highPriorityLevel = highPriorityLevel;
    this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
    this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
    this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
    this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout",
                                     2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);

    // Start the listener here and let it bind to the port
	//Reactor主线程,接受connect请求
    listener = new Listener();
    ......
    // Create the responder here
	//写回Response的线程,异步写
    responder = new Responder();
  }

 Listener初始化

public Listener() throws IOException {
      address = new InetSocketAddress(bindAddress, port);
      // Create a new server socket and set to non blocking mode
	//打开一个Server端Channel,非阻塞模式
      acceptChannel = ServerSocketChannel.open();
      acceptChannel.configureBlocking(false);

      // Bind the server socket to the local host and port
	//Bind端口
      bind(acceptChannel.socket(), address, backlogLength);
      port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
      // create a selector;
	//Listener主线程的selector
      selector= Selector.open();
	//Reactor的子线程,负责对通道进行读取和处理,一个Reader可同时处理多个链接
      readers = new Reader[readThreads];
	//Reader线程池
      readPool = Executors.newFixedThreadPool(readThreads,
        new ThreadFactoryBuilder().setNameFormat(
          "IPC Reader %d on port " + port).setDaemon(true).build());
      for (int i = 0; i < readThreads; ++i) {
        Reader reader = new Reader();
        readers[i] = reader;
        readPool.execute(reader);
      }

      // Register accepts on the server socket with the selector.
	//注册ACCEPT事件,selector可以处理链接请求
      acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
      this.setName("IPC Server listener on " + port);
      this.setDaemon(true);
    }

 Responder初始化

   Responder() throws IOException {
      this.setName("IPC Server Responder");
      this.setDaemon(true);
	//先打开一个Selector,后续需要写回数据的通道会注册WRITE事件上来
      writeSelector = Selector.open(); // create a selector
      pending = 0;
    }

 初始化完成之后,RegionServer进行启动前的其他操作,比如新建zk链接,处理master,启动flush等线程。最终会启动RPC服务。

HBaseServer启动

  public void start() {
    startThreads();
    openServer();
  }

 

public synchronized void startThreads() {
	//Responder线程启动
    responder.start();
	//Listener线程启动
    listener.start();
	//Handler线程启动,默认10
    handlers = new Handler[handlerCount];

    for (int i = 0; i < handlerCount; i++) {
      handlers[i] = new Handler(callQueue, i);
      handlers[i].start();
    }

    if (priorityHandlerCount > 0) {
      priorityHandlers = new Handler[priorityHandlerCount];
      for (int i = 0 ; i < priorityHandlerCount; i++) {
        priorityHandlers[i] = new Handler(priorityCallQueue, i);
        priorityHandlers[i].start();
      }
    }
  }

 Responder启动,Responder负责RPC响应写回操作,异步写

 while (running) {
        try {
          waitPending();     // If a channel is being registered, wait.
		//writeSelector监听OS WRITE READY的事件
          writeSelector.select(purgeTimeout);
          Iterator iter = writeSelector.selectedKeys().iterator();
          while (iter.hasNext()) {
            SelectionKey key = iter.next();
            iter.remove();
            try {
              if (key.isValid() && key.isWritable()) {
			//异步写回,如果一次没写完,继续监听对应channel的WRITE事件,继续写
                  doAsyncWrite(key);
              }
            } catch (IOException e) {
              LOG.info(getName() + ": doAsyncWrite threw exception " + e);
            }
          }
          long now = System.currentTimeMillis();
          if (now < lastPurgeTime + purgeTimeout) {
            continue;
          }
          lastPurgeTime = now;
          //
          // If there were some calls that have not been sent out for a
          // long time, discard them.
          //
          LOG.debug("Checking for old call responses.");
		//长时间还没写完的请求,清理之,并关闭链接
          ArrayList calls;

          // get the list of channels from list of keys.
          synchronized (writeSelector.keys()) {
            calls = new ArrayList(writeSelector.keys().size());
            iter = writeSelector.keys().iterator();
            while (iter.hasNext()) {
              SelectionKey key = iter.next();
              Call call = (Call)key.attachment();
              if (call != null && key.channel() == call.connection.channel) {
                calls.add(call);
              }
            }
          }

          for(Call call : calls) {
            try {
              doPurge(call, now);
            } catch (IOException e) {
              LOG.warn("Error in purging old calls " + e);
            }
          }
        .......
      }

 写数据过程

    

private void doAsyncWrite(SelectionKey key) throws IOException {
      Call call = (Call)key.attachment();
      if (call == null) {
        return;
      }
      if (key.channel() != call.connection.channel) {
        throw new IOException("doAsyncWrite: bad channel");
      }

      synchronized(call.connection.responseQueue) {
	//如果这个Connection的数据写完了,则清理key
        if (processResponse(call.connection.responseQueue, false)) {
          try {
            key.interestOps(0);
          } catch (CancelledKeyException e) {
            /* The Listener/reader might have closed the socket.
             * We don't explicitly cancel the key, so not sure if this will
             * ever fire.
             * This warning could be removed.
             */
            LOG.warn("Exception while changing ops : " + e);
          }
        }
      }
    }

 

 

private boolean processResponse(final LinkedList responseQueue,
                                    boolean inHandler) throws IOException {
      boolean error = true;
      boolean done = false;       // there is more data for this channel.
      int numElements;
      Call call = null;
      try {
        //noinspection SynchronizationOnLocalVariableOrMethodParameter
        synchronized (responseQueue) {
          //
          // If there are no items for this channel, then we are done
          //
          numElements = responseQueue.size();
          if (numElements == 0) {
            error = false;
            return true;              // no more data for this channel.
          }
          //
          // Extract the first call
          //写请求
          call = responseQueue.removeFirst();
          SocketChannel channel = call.connection.channel;
          if (LOG.isDebugEnabled()) {
            LOG.debug(getName() + ": responding to #" + call.id + " from " +
                      call.connection);
          }
          //
          // Send as much data as we can in the non-blocking fashion
          //异步写回
          int numBytes = channelWrite(channel, call.response);
          if (numBytes < 0) {
            return true;
          }
		//如果写完了,递减RPC统计数目
          if (!call.response.hasRemaining()) {
            call.connection.decRpcCount();
            //noinspection RedundantIfStatement
            if (numElements == 1) {    // last call fully processes.
              done = true;             // no more data for this channel.
            } else {
              done = false;            // more calls pending to be sent.
            }
            if (LOG.isDebugEnabled()) {
              LOG.debug(getName() + ": responding to #" + call.id + " from " +
                        call.connection + " Wrote " + numBytes + " bytes.");
            }
          } 
	//如果没写完,放回去,下次继续写
	else {
            //
            // If we were unable to write the entire response out, then
            // insert in Selector queue.
            //
            call.connection.responseQueue.addFirst(call);

           ......
          }
          error = false;              // everything went off well
        }
      } finally {
        if (error && call != null) {
          LOG.warn(getName()+", call " + call + ": output error");
          done = true;               // error. no more data for this channel.
          closeConnection(call.connection);
        }
      }
      return done;
    }

 具体channelWrite

  protected int channelWrite(WritableByteChannel channel,
                                    ByteBuffer buffer) throws IOException {
	//如果小于BUFFER,直接写,否则分批写
    int count =  (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
           channel.write(buffer) : channelIO(null, channel, buffer);
    if (count > 0) {
      rpcMetrics.sentBytes.inc(count);
    }
    return count;
  }

 分批写

  private static int channelIO(ReadableByteChannel readCh,
                               WritableByteChannel writeCh,
                               ByteBuffer buf) throws IOException {

    int originalLimit = buf.limit();
    int initialRemaining = buf.remaining();
    int ret = 0;
	//没写完,一直写
    while (buf.remaining() > 0) {
      try {
	//每次要写的数量,分批
        int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
	//写之前,写调整limit到目标index,写回的时候是取position和limit之间的数据
        buf.limit(buf.position() + ioSize);
	//异步写,有可能只写了部分数据
        ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
	//只写了部分数据,说明OS IO很繁忙,直接返回,下次继续写
        if (ret < ioSize) {
          break;
        }

      } 
	//每批数据写完,将limit重置
	finally {
        buf.limit(originalLimit);
      }
    }
	//返回写了多少数据
    int nBytes = initialRemaining - buf.remaining();
    return (nBytes > 0) ? nBytes : ret;
  }

 Listener启动

  while (running) {
        SelectionKey key = null;
        try {
		//阻塞select,ACCEPT并发不高
          selector.select(); // FindBugs IS2_INCONSISTENT_SYNC
          Iterator iter = selector.selectedKeys().iterator();
          while (iter.hasNext()) {
            key = iter.next();
            iter.remove();
            try {
              if (key.isValid()) {
                if (key.isAcceptable())
                  doAccept(key);
              }
            } catch (IOException ignored) {
            }
            key = null;
          }
        ......
      }

 accept过程

void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {
      Connection c;
      ServerSocketChannel server = (ServerSocketChannel) key.channel();

      SocketChannel channel;
	//ACCEPT成功,则分配给Reader处理具体IO
      while ((channel = server.accept()) != null) {
        channel.configureBlocking(false);
        channel.socket().setTcpNoDelay(tcpNoDelay);
        channel.socket().setKeepAlive(tcpKeepAlive);
	//顺序拿Reader
        Reader reader = getReader();
        try {
          reader.startAdd();
		//注册,READ事件到selector
          SelectionKey readKey = reader.registerChannel(channel);
		//业务层面的Connection,放入attach共享
          c = getConnection(channel, System.currentTimeMillis());
          readKey.attach(c);
	//当前链接
          synchronized (connectionList) {
            connectionList.add(numConnections, c);
            numConnections++;
          }
          ......
        } finally {
          reader.finishAdd();
        }
      }
      rpcMetrics.numOpenConnections.set(numConnections);
    }

 accept之后,可以进行IO操作,Reader处理

      private synchronized void doRunLoop() {
        while (running) {
          SelectionKey key = null;
          try {
		//同步读
            readSelector.select();
		//有新链接进来,等之
            while (adding) {
              this.wait(1000);
            }
            alive = true;
            Iterator iter = readSelector.selectedKeys().iterator();
            while (iter.hasNext()) {
              key = iter.next();
              iter.remove();
              if (key.isValid()) {
                if (key.isReadable()) {
		//读就位
                  doRead(key);
                }
              }
              key = null;
            }
          } catch (InterruptedException e) {
            if (running) {                      // unexpected -- log it
              LOG.info(getName() + " unexpectedly interrupted: " +
                  StringUtils.stringifyException(e));
            }
          } catch (IOException ex) {
            LOG.error("Error in Reader", ex);
          } finally {
            alive = false;
          }
        }
      }

 具体读

void doRead(SelectionKey key) throws InterruptedException {
      int count = 0;
      Connection c = (Connection)key.attachment();
      if (c == null) {
        return;
      }
      c.setLastContact(System.currentTimeMillis());

      try {
	//业务逻辑都封装在Connection里
        count = c.readAndProcess();
      } catch (InterruptedException ieo) {
        throw ieo;
      } catch (Exception e) {
        LOG.warn(getName() + ": readAndProcess threw exception " + e + ". Count of bytes read: " + count, e);
        count = -1; //so that the (count < 0) block is executed
      }
      if (count < 0) {
        if (LOG.isDebugEnabled())
          LOG.debug(getName() + ": disconnecting client " +
                    c.getHostAddress() + ". Number of active connections: "+
                    numConnections);
        closeConnection(c);
        // c = null;
      }
      else {
        c.setLastContact(System.currentTimeMillis());
      }
    }

 Connection的read

public int readAndProcess() throws IOException, InterruptedException {
      while (true) {
        /* Read at most one RPC. If the header is not read completely yet
         * then iterate until we read first RPC or until there is no data left.
         */
        int count;
	//读int,4个字节,代表data长度
        if (dataLengthBuffer.remaining() > 0) {
          count = channelRead(channel, dataLengthBuffer);
          if (count < 0 || dataLengthBuffer.remaining() > 0)
            return count;
        }
	//version还没读,意味着这个client是刚连上来,则读version,一个字节
        if (!versionRead) {
          //Every connection is expected to send the header.
          ByteBuffer versionBuffer = ByteBuffer.allocate(1);
          count = channelRead(channel, versionBuffer);
          if (count <= 0) {
            return count;
          }
          int version = versionBuffer.get(0);

          dataLengthBuffer.flip();
	//前四个字节固定,或者版本不对,返回异常
          if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) {
            //Warning is ok since this is not supposed to happen.
            LOG.warn("Incorrect header or version mismatch from " +
                     hostAddress + ":" + remotePort +
                     " got version " + version +
                     " expected version " + CURRENT_VERSION);
            setupBadVersionResponse(version);
            return -1;
          }
	//HEADER读完,继续下一个请求
          dataLengthBuffer.clear();
          versionRead = true;
          continue;
        }
	//根据data length,读data
        if (data == null) {
          dataLengthBuffer.flip();
          dataLength = dataLengthBuffer.getInt();
	//-1代表ping请求
          if (dataLength == HBaseClient.PING_CALL_ID) {
            dataLengthBuffer.clear();
            return 0;  //ping message
          }
		//分配内存
          data = ByteBuffer.allocate(dataLength);
          incRpcCount();  // Increment the rpc count
        }
	//读入数据
        count = channelRead(channel, data);
	//读满了,继续业务处理,否则直接返回,下次继续读,一直读满为止
        if (data.remaining() == 0) {
          dataLengthBuffer.clear();
          data.flip();
	//header之后是业务请求
          if (headerRead) {
            processData(data.array());
            data = null;
            return count;
          }
	//读header信息,主要是初始化Connection的protocol属性
          processHeader();
          headerRead = true;
          data = null;
          continue;
        }
        return count;
      }
    }

 请求处理

protected void processData(byte[] buf) throws  IOException, InterruptedException {
      DataInputStream dis =
        new DataInputStream(new ByteArrayInputStream(buf));
	//请求的客户端id
      int id = dis.readInt();                    // try to read an id
	//请求大小      
      long callSize = buf.length;
	......

      // Enforcing the call queue size, this triggers a retry in the client
	//大小超过限制,则返回异常 
      if ((callSize + callQueueSize.get()) > maxQueueSize) {
        final Call callTooBig =
          new Call(id, null, this, responder, callSize);
        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
        setupResponse(responseBuffer, callTooBig, Status.FATAL, null,
            IOException.class.getName(),
            "Call queue is full, is ipc.server.max.callqueue.size too small?");
        responder.doRespond(callTooBig);
        return;
      }
	//param对象,默认Invocation
      Writable param;
      try {
        param = ReflectionUtils.newInstance(paramClass, conf);//read param
        param.readFields(dis);
      } catch (Throwable t) {
        LOG.warn("Unable to read call parameters for client " +
                 getHostAddress(), t);
        final Call readParamsFailedCall =
          new Call(id, null, this, responder, callSize);
        ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();

        setupResponse(responseBuffer, readParamsFailedCall, Status.FATAL, null,
            t.getClass().getName(),
            "IPC server unable to read call parameters: " + t.getMessage());
        responder.doRespond(readParamsFailedCall);
        return;
      }
	//将请求添加到处理队列,后续Handler线程处理
      Call call = new Call(id, param, this, responder, callSize);
	//递增请求队列大小
      callQueueSize.add(callSize);

      if (priorityCallQueue != null && getQosLevel(param) > highPriorityLevel) {
        priorityCallQueue.put(call);
        updateCallQueueLenMetrics(priorityCallQueue);
      } else {
        callQueue.put(call);              // queue the call; maybe blocked here
        updateCallQueueLenMetrics(callQueue);
      }
    }

 Invocation反序列化

    public void readFields(DataInput in) throws IOException {
	//请求方法名
      methodName = UTF8.readString(in);
	//参数个数
      parameters = new Object[in.readInt()];
	//对应类型
      parameterClasses = new Class[parameters.length];
      ObjectWritable objectWritable = new ObjectWritable();
	//每个参数根据不同类型,反序列化
      for (int i = 0; i < parameters.length; i++) {
        parameters[i] = ObjectWritable.readObject(in, objectWritable, this.conf);
        parameterClasses[i] = objectWritable.getDeclaredClass();
      }
    }

 序列化过程,支持原生类型,字符串,Writable实现,和数组

 public static Object readObject(DataInput in, ObjectWritable objectWritable, Configuration conf)
    throws IOException {
	//类型->class对象
    String className = UTF8.readString(in);
    Class declaredClass = PRIMITIVE_NAMES.get(className);
    if (declaredClass == null) {
      try {
        declaredClass = conf.getClassByName(className);
      } catch (ClassNotFoundException e) {
        throw new RuntimeException("readObject can't find class " + className, e);
      }
    }    

    Object instance;
    //原生类型
    if (declaredClass.isPrimitive()) {            // primitive types

      if (declaredClass == Boolean.TYPE) {             // boolean
        instance = Boolean.valueOf(in.readBoolean());
      } else if (declaredClass == Character.TYPE) {    // char
        instance = Character.valueOf(in.readChar());
      } else if (declaredClass == Byte.TYPE) {         // byte
        instance = Byte.valueOf(in.readByte());
      } else if (declaredClass == Short.TYPE) {        // short
        instance = Short.valueOf(in.readShort());
      } else if (declaredClass == Integer.TYPE) {      // int
        instance = Integer.valueOf(in.readInt());
      } else if (declaredClass == Long.TYPE) {         // long
        instance = Long.valueOf(in.readLong());
      } else if (declaredClass == Float.TYPE) {        // float
        instance = Float.valueOf(in.readFloat());
      } else if (declaredClass == Double.TYPE) {       // double
        instance = Double.valueOf(in.readDouble());
      } else if (declaredClass == Void.TYPE) {         // void
        instance = null;
      } else {
        throw new IllegalArgumentException("Not a primitive: "+declaredClass);
      }

    } 
	//递归读数组
    else if (declaredClass.isArray()) {              // array
      int length = in.readInt();
      instance = Array.newInstance(declaredClass.getComponentType(), length);
      for (int i = 0; i < length; i++) {
        Array.set(instance, i, readObject(in, conf));
      }
      
    } else if (declaredClass == String.class) {        // String
      instance = UTF8.readString(in);
    } else if (declaredClass.isEnum()) {         // enum
      instance = Enum.valueOf((Class) declaredClass, UTF8.readString(in));
    } 
	//Writable
    else {                                      // Writable
      Class instanceClass = null;
      String str = "";
	//实现类名
      try {
        str = UTF8.readString(in);
        instanceClass = conf.getClassByName(str);
      } catch (ClassNotFoundException e) {
        throw new RuntimeException("readObject can't find class " + str, e);
      }
      //Writable实现类需要自己实现反序列化
      Writable writable = WritableFactories.newInstance(instanceClass, conf);
      writable.readFields(in);
      instance = writable;

      if (instanceClass == NullInstance.class) {  // null
        declaredClass = ((NullInstance)instance).declaredClass;
        instance = null;
      }
    }

    if (objectWritable != null) {                 // store values
      objectWritable.declaredClass = declaredClass;
      objectWritable.instance = instance;
    }

    return instance;
      
  }

 之后就是Handler线程处理了

public void run() {
      LOG.info(getName() + ": starting");
      status.setStatus("starting");
      SERVER.set(HBaseServer.this);
      while (running) {
        try {
          status.pause("Waiting for a call");
	//从queue中拿请求
          Call call = myCallQueue.take(); // pop the queue; maybe blocked here
          updateCallQueueLenMetrics(myCallQueue);
          status.setStatus("Setting up call");
          status.setConnection(call.connection.getHostAddress(), 
              call.connection.getRemotePort());

          if (LOG.isDebugEnabled())
            LOG.debug(getName() + ": has #" + call.id + " from " +
                      call.connection);

          String errorClass = null;
          String error = null;
          Writable value = null;
	//ThreadLocal请求
          CurCall.set(call);
          alive = true;
          try {
            if (!started)
              throw new ServerNotRunningYetException("Server is not running yet");

        ......

            RequestContext.set(call.connection.ticket, getRemoteIp(),
                call.connection.protocol);
            // make the call
		//反射调用对应服务,返回结果
            value = call(call.connection.protocol, call.param, call.timestamp, 
                status);
          } catch (Throwable e) {
            LOG.debug(getName()+", call "+call+": error: " + e, e);
            errorClass = e.getClass().getName();
            error = StringUtils.stringifyException(e);
          } finally {
            alive = false;
            // Must always clear the request context to avoid leaking
            // credentials between requests.
            RequestContext.clear();
          }
          CurCall.set(null);
		//减小queue大小
          callQueueSize.add(call.getSize() * -1);
          // Set the response for undelayed calls and delayed calls with
          // undelayed responses.
		//序列化写回response
          if (!call.isDelayed() || !call.isReturnValueDelayed()) {
            call.setResponse(value,
              errorClass == null? Status.SUCCESS: Status.ERROR,
                errorClass, error);
          }
		//通过responder写回响应
          call.sendResponseIfReady();
          status.markComplete("Sent response");
        } 
	.......
    }

  }

 反射调用过程

public Writable call(Class protocol,
        Writable param, long receivedTime, MonitoredRPCHandler status)
    throws IOException {
      try {
        Invocation call = (Invocation)param;
        if(call.getMethodName() == null) {
          throw new IOException("Could not find requested method, the usual " +
              "cause is a version mismatch between client and server.");
        }
        if (verbose) log("Call: " + call);
        status.setRPC(call.getMethodName(), call.getParameters(), receivedTime);
        status.setRPCPacket(param);
        status.resume("Servicing call");
	//method对象
        Method method =
          protocol.getMethod(call.getMethodName(),
                                   call.getParameterClasses());
	//make invokable
        method.setAccessible(true);

        //Verify protocol version.
        //Bypass the version check for VersionedProtocol
        if (!method.getDeclaringClass().equals(VersionedProtocol.class)) {
          long clientVersion = call.getProtocolVersion();
          ProtocolSignature serverInfo = ((VersionedProtocol) instance)
              .getProtocolSignature(protocol.getCanonicalName(), call
                  .getProtocolVersion(), call.getClientMethodsHash());
          long serverVersion = serverInfo.getVersion();
          if (serverVersion != clientVersion) {
            LOG.warn("Version mismatch: client version=" + clientVersion
                + ", server version=" + serverVersion);
            throw new RPC.VersionMismatch(protocol.getName(), clientVersion,
                serverVersion);
          }
        }
        Object impl = null;
	//实现类是HRegionServer
        if (protocol.isAssignableFrom(this.implementation)) {
          impl = this.instance;
        }
        else {
          throw new HBaseRPC.UnknownProtocolException(protocol);
        }

        long startTime = System.currentTimeMillis();
        Object[] params = call.getParameters();
	//反射调用
        Object value = method.invoke(impl, params);
        int processingTime = (int) (System.currentTimeMillis() - startTime);
        int qTime = (int) (startTime-receivedTime);
        ......
        rpcMetrics.rpcQueueTime.inc(qTime);
        rpcMetrics.rpcProcessingTime.inc(processingTime);
        rpcMetrics.inc(call.getMethodName(), processingTime);
        if (verbose) log("Return: "+value);
	//包装返回对象
        HbaseObjectWritable retVal =
          new HbaseObjectWritable(method.getReturnType(), value);
        long responseSize = retVal.getWritableSize();
        // log any RPC responses that are slower than the configured warn
        // response time or larger than configured warning size
        boolean tooSlow = (processingTime > warnResponseTime
            && warnResponseTime > -1);
        boolean tooLarge = (responseSize > warnResponseSize
            && warnResponseSize > -1);
        if (tooSlow || tooLarge) {
          // when tagging, we let TooLarge trump TooSmall to keep output simple
          // note that large responses will often also be slow.
          logResponse(call, (tooLarge ? "TooLarge" : "TooSlow"),
              status.getClient(), startTime, processingTime, qTime,
              responseSize);
          // provides a count of log-reported slow responses
          if (tooSlow) {
            rpcMetrics.rpcSlowResponseTime.inc(processingTime);
          }
        }
        if (processingTime > 1000) {
          // we use a hard-coded one second period so that we can clearly
          // indicate the time period we're warning about in the name of the 
          // metric itself
          rpcMetrics.inc(call.getMethodName() + ABOVE_ONE_SEC_METRIC,
              processingTime);
        }

        return retVal;
      } catch (InvocationTargetException e) {
        Throwable target = e.getTargetException();
        if (target instanceof IOException) {
          throw (IOException)target;
        }
        IOException ioe = new IOException(target.toString());
        ioe.setStackTrace(target.getStackTrace());
        throw ioe;
      } catch (Throwable e) {
        if (!(e instanceof IOException)) {
          LOG.error("Unexpected throwable object ", e);
        }
        IOException ioe = new IOException(e.toString());
        ioe.setStackTrace(e.getStackTrace());
        throw ioe;
      }
    }

 响应对象序列化

protected synchronized void setResponse(Object value, Status status,
        String errorClass, String error) {
      // Avoid overwriting an error value in the response.  This can happen if
      // endDelayThrowing is called by another thread before the actual call
      // returning.
      if (this.isError)
        return;
      if (errorClass != null) {
        this.isError = true;
      }
      Writable result = null;
      if (value instanceof Writable) {
        result = (Writable) value;
      } else {
        /* We might have a null value and errors. Avoid creating a
         * HbaseObjectWritable, because the constructor fails on null. */
        if (value != null) {
          result = new HbaseObjectWritable(value);
        }
      }
	//序列化大小
      int size = BUFFER_INITIAL_SIZE;
      if (result instanceof WritableWithSize) {
        // get the size hint.
        WritableWithSize ohint = (WritableWithSize) result;
        long hint = ohint.getWritableSize() + Bytes.SIZEOF_BYTE +
          (2 * Bytes.SIZEOF_INT);
        if (hint > Integer.MAX_VALUE) {
          // oops, new problem.
          IOException ioe =
            new IOException("Result buffer size too large: " + hint);
          errorClass = ioe.getClass().getName();
          error = StringUtils.stringifyException(ioe);
        } else {
          size = (int)hint;
        }
      }

      ByteBufferOutputStream buf = new ByteBufferOutputStream(size);
      DataOutputStream out = new DataOutputStream(buf);
      try {
        // Call id.
	//客户端请求id
        out.writeInt(this.id);
        // Write flag.
	//异常标示
        byte flag = (error != null)?
          ResponseFlag.getErrorAndLengthSet(): ResponseFlag.getLengthSetOnly();
        out.writeByte(flag);
        // Place holder for length set later below after we
        // fill the buffer with data.
	//长度占位
        out.writeInt(0xdeadbeef);
	//处理结果
        out.writeInt(status.state);
      } catch (IOException e) {
        errorClass = e.getClass().getName();
        error = StringUtils.stringifyException(e);
      }

      try {
	//序列化响应对象
        if (error == null) {
          result.write(out);
        } 
	//异常信息
	else {
          WritableUtils.writeString(out, errorClass);
          WritableUtils.writeString(out, error);
        }
      } catch (IOException e) {
        LOG.warn("Error sending response to call: ", e);
      }

      // Set the length into the ByteBuffer after call id and after
      // byte flag.
      ByteBuffer bb = buf.getByteBuffer();
	//数据总大小
      int bufSiz = bb.remaining();
      // Move to the size location in our ByteBuffer past call.id
      // and past the byte flag.
	//长度占位字节填充
      bb.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE); 
      bb.putInt(bufSiz);
      bb.position(0);
      this.response = bb;
    }

 最终数据被写回client,client部分代码见下一篇

你可能感兴趣的:(hbase,rpc)