The Message Process of Giraph

AbstractComputation.sendMessage

 /**
   * Send a message to a vertex id.
   *
   * @param id Vertex id to send the message to
   * @param message Message data to send
   */
@Override
  public void sendMessage(I id, M2 message) {
    workerClientRequestProcessor.sendMessageRequest(id, message);
  }

NettyWorkerClientRequestProcessor.sendMessageRequest

  @Override
  public void sendMessageRequest(I destVertexId, Writable message) {
    this.sendMessageCache.sendMessageRequest(destVertexId, message);
  }

SendMessageCache.sendMessageRequest

SendMessageCache extends SendVertexIdDataCache.

public void sendMessageRequest(I destVertexId, M message) {
    PartitionOwner owner =
      getServiceWorker().getVertexPartitionOwner(destVertexId);
    WorkerInfo workerInfo = owner.getWorkerInfo();
    final int partitionId = owner.getPartitionId();
   
    ++totalMsgsSentInSuperstep;
    // Add the message to the cache
    int workerMessageSize = addMessage(
      workerInfo, partitionId, destVertexId, message);
   
    if (workerMessageSize >= maxMessagesSizePerWorker) {
      PairList>
        workerMessages = removeWorkerMessages(workerInfo);
      WritableRequest writableRequest =
        new SendWorkerMessagesRequest(workerMessages);
      totalMsgBytesSentInSuperstep += writableRequest.getSerializedSize();
      clientProcessor.doRequest(workerInfo, writableRequest);
      // Notify sending
      getServiceWorker().getGraphTaskManager().notifySentMessages();
    }
  }

SendMessageCache.addMessage

  public int addMessage(WorkerInfo workerInfo,
                        int partitionId, I destVertexId, M message) {
    return addData(workerInfo, partitionId, destVertexId, message);
  }

SendVertexIdDataCache.addData

  public int addData(WorkerInfo workerInfo,
                     int partitionId, I destVertexId, T data) {
    // Get the data collection
    VertexIdData partitionData =
        getPartitionData(workerInfo, partitionId);
    int originalSize = partitionData.getSize();
    partitionData.add(destVertexId, data);
    // Update the size of cached, outgoing data per worker
    return incrDataSize(workerInfo.getTaskId(),
        partitionData.getSize() - originalSize);
  }

SendVertexIdDataCache.getPartitionData

 private VertexIdData getPartitionData(WorkerInfo workerInfo,
                                                       int partitionId) {
    // Get the data collection
    B partitionData = getData(partitionId);
    if (partitionData == null) {
      partitionData = createVertexIdData();
      partitionData.setConf(getConf());
      partitionData.initialize(getInitialBufferSize(workerInfo.getTaskId()));
      setData(partitionId, partitionData);
    }

    return partitionData;
  }

SendMessageCache.getData

private final D[] dataCache;
 public D getData(int partitionId) {
    return dataCache[partitionId];
  }

The type of partitionData is ByteArrayVertexIdMessages. ByteArrayVertexIdMessages stores vertex id and message pairs in a single byte array.

ByteArrayVertexIdMessages.initialize

create byte array.

public void initialize(int expectedSize) {
    extendedDataOutput = getConf().createExtendedDataOutput(expectedSize);
  }

ByteArrayVertexIdMessages.add

write contents to extendedDataOutput.

 @Override
  public void add(I vertexId, T data) {
    try {
      vertexId.write(extendedDataOutput);
      writeData(extendedDataOutput, data);
    } catch (IOException e) {
      throw new IllegalStateException("add: IOException", e);
    }
  }

send message out

maxMessagesSizePerWorker: 524288

if (workerMessageSize >= maxMessagesSizePerWorker) {
      PairList>
        workerMessages = removeWorkerMessages(workerInfo);
      WritableRequest writableRequest =
        new SendWorkerMessagesRequest(workerMessages);
      totalMsgBytesSentInSuperstep += writableRequest.getSerializedSize();
      clientProcessor.doRequest(workerInfo, writableRequest);
      // Notify sending
      getServiceWorker().getGraphTaskManager().notifySentMessages();
    }

The SendWorkerMessagesRequest.doRequest will be called eventually.

SendWorkerMessagesRequest.doRequest

  @Override
  public void doRequest(ServerData serverData) {
    PairList>.Iterator
        iterator = partitionVertexData.getIterator();
    while (iterator.hasNext()) {
      iterator.next();
      serverData.getIncomingMessageStore().
          addPartitionMessages(iterator.getCurrentFirst(),
              iterator.getCurrentSecond());
    }
  }

DiskBackedMessageStore.addPartitionMessages

If out of core is used, DiskBackedMessageStore.addPartitionMessages will be called…

  public void addPartitionMessages(
      int partitionId, VertexIdMessages messages) {
    if (useMessageCombiner) {
      messageStore.addPartitionMessages(partitionId, messages);
    } else {
      addEntry(partitionId, messages);
    }
  }

@Override
  public void addPartitionMessages(
      int partitionId, VertexIdMessages messages) {
    if (useMessageCombiner) {
      messageStore.addPartitionMessages(partitionId, messages);
    } else {
      addEntry(partitionId, messages);
    }
  }

LongDoubleMessageStore.addPartitionMessages

public void addPartitionMessages(int partitionId,
      VertexIdMessages messages) {
    LongWritable reusableVertexId = new LongWritable();
    DoubleWritable reusableMessage = new DoubleWritable();
    DoubleWritable reusableCurrentMessage = new DoubleWritable();

    Long2DoubleOpenHashMap partitionMap = map.get(partitionId);
    synchronized (partitionMap) {
      VertexIdMessageIterator iterator =
        messages.getVertexIdMessageIterator();
      while (iterator.hasNext()) {
        iterator.next();
        long vertexId = iterator.getCurrentVertexId().get();
        double message = iterator.getCurrentMessage().get();
        if (partitionMap.containsKey(vertexId)) {
          reusableVertexId.set(vertexId);
          reusableMessage.set(message);
          reusableCurrentMessage.set(partitionMap.get(vertexId));
          messageCombiner.combine(reusableVertexId, reusableCurrentMessage,
              reusableMessage);
          message = reusableCurrentMessage.get();
        }
        // FIXME: messageCombiner should create an initial message instead
        partitionMap.put(vertexId, message);
      }
    }
  }