Apollo 3.5 Cyber 多進程通訊模塊 - Transport (Shared Memory篇）
Apollo 3.5 Cyber 多進程通訊模塊 - Transport (Shared Memory篇）
- 每個Class的功用
- 由`Transmitter`發新數據到`Receiver`處理數據的流程
- shared memory的syscall
ROS為不個的node之間的通訊提供了shared memory和network兩種方法 (是也有一個叫intra, 不過那不能cross-process的。而cyber的Node::Reader和Node::Writer是預設三種都用的，即message會視乎情況選對應的方法去發，也可能用三種方法都發一次，也就是hybrid mode了）
Cyber也提供了差不多的東西，今天就看一看
先上一個全局的圖。建議放大看。
當中主要有ShmDispatcher, ShmReceiver, ShmTransmitter,Segment,NotifierBase這幾個。其他的就是實現的細節。當你了解這幾個主要class的互動時，就會基本了解其機制了。
每個Class的功用
Segment: 負責管理一段Shared Memory, 提供Accquire-Release的接口讓其他class可以在thread-safe的前題下拿到shared memory中的對象. 而因為Segment的Block(s)是放在shared memory的，就算在同一線程中生成同一channel的多個segment,其內容空間也是一樣的Block: 一段Segment中有多個Block,channel中的數據會存在其中，而Block也是其他class用來讀寫數據的對象State: 管理Segment的內部狀態。因為Segment應該是在多個process中都是有相同狀態的，所以要特別抽出來放到shared memory中去肯定同步。Receiver:ShmReceiver的Base class, 它下邊除了ShmReceiver，還有RtpsReceiver,IntraReceiver等等。主要就是負責`當指定channel有數據時，調用指定的回調，並提供一個開關（是否繼續監聽channel）的功能“ShmReceiver:Receiver的shared memory版實現，在channel中的數據會利用shared memory去共享Transmitter:ShmTransmitter的Base class, 它下邊也是有shared memory版本，rtps版本，等等。主要就是負責`提供一個接口，容許你發數據到指定channel，並告知對應Notifier有更新。並提供一個開關（是否讓其內部指針指到Notifier的singleton,這沒有甚麼用，反正不發數據就和關了差不多）的功能“ShmTransmitter的shared memory版實現Dispatcher: 用來管理所有的channel的讀數據的部份。當有新數據時就調用Receiver註冊的回調。其subclass都是Singleton。ShmDispatcher: 做polling,不停去看Notifier有沒有新的info過來，如果有新info來了，就解讀info，在對應Segment中拿數據及把其deserialize為一個對象.最後回調Receiver的回調Notifier:發新數據跟發有新數據的提示是兩個不同的概念，Dispatcher做了前者，而Notifier做了後者,而Notifier也提供了多種實現方式.Notifier的所有subclass也是單例MulticastNotifier: 用了multicast頻道，用socket通訊實現的NotifierConditionNotifier: 用conditional variable在shared memory實現的NotifierNotifierFactory: 按全局config,生成MulticastNotifier或ConditionNotifierReadableInfo:Notifier在有新數據時所發出的結構體,包含了以下幾個資料- 是那個channel
- 是誰發的
- 第幾個block
ListenHandler: 基本就是callback, 不過加上了一些額外的資訊，容訊做到一些額外功能。比如只有在某一個特定node發數據時才調用。Signal,Conenct,Slot就一個通用的回調機制
由Transmitter發新數據到Receiver處理數據的流程
其實看完上邊應該都了解得差不多了，不過都寫一寫
- 調用
Transmitter的Transmit接口去發數據
// cyber/transport/transmitter/transmitter.h
virtual bool Transmit(const MessagePtr& msg);
virtual bool Transmit(const MessagePtr& msg, const MessageInfo& msg_info) = 0;
ShmTransmitter版的Transmit實現是先從Segment中拿到可以寫的Block，然後寫進去。最後告訴Notifier，有新message了
template <typename M>
bool ShmTransmitter<M>::Transmit(const MessagePtr& msg,
const MessageInfo& msg_info) {
return Transmit(*msg, msg_info);
}
template <typename M>
bool ShmTransmitter<M>::Transmit(const M& msg, const MessageInfo& msg_info) {
if (!this->enabled_) {
ADEBUG << "not enable.";
return false;
}
WritableBlock wb;
std::size_t msg_size = message::ByteSize(msg);
if (!segment_->AcquireBlockToWrite(msg_size, &wb)) {
AERROR << "acquire block failed.";
return false;
}
ADEBUG << "block index: " << wb.index;
if (!message::SerializeToArray(msg, wb.buf, static_cast<int>(msg_size))) {
AERROR << "serialize to array failed.";
segment_->ReleaseWrittenBlock(wb);
return false;
}
wb.block->set_msg_size(msg_size);
char* msg_info_addr = reinterpret_cast<char*>(wb.buf) + msg_size;
if (!msg_info.SerializeTo(msg_info_addr, MessageInfo::kSize)) {
AERROR << "serialize message info failed.";
segment_->ReleaseWrittenBlock(wb);
return false;
}
wb.block->set_msg_info_size(MessageInfo::kSize);
segment_->ReleaseWrittenBlock(wb);
ReadableInfo readable_info(host_id_, wb.index, channel_id_);
ADEBUG << "Writing sharedmem message: "
<< common::GlobalData::GetChannelById(channel_id_)
<< " to block: " << wb.index;
return notifier_->Notify(readable_info);
}
- 我們這裹先假設是用
ConditionNotifier,Notify會通過indicator_->cv.notify_all();讓Listen返回true
// cyber/transport/shm/condition_notifier.cc
bool ConditionNotifier::Notify(const ReadableInfo& info) {
if (is_shutdown_.load()) {
ADEBUG << "notifier is shutdown.";
return false;
}
{
std::unique_lock<std::mutex> lck(indicator_->mtx);
auto idx = indicator_->written_info_num % kBufLength;
indicator_->infos[idx] = info;
++indicator_->written_info_num;
}
indicator_->cv.notify_all();
return true;
}
bool ConditionNotifier::Listen(int timeout_ms, ReadableInfo* info) {
if (info == nullptr) {
AERROR << "info nullptr.";
return false;
}
if (is_shutdown_.load()) {
ADEBUG << "notifier is shutdown.";
return false;
}
std::unique_lock<std::mutex> lck(indicator_->mtx);
if (next_listen_num_ >= indicator_->written_info_num) {
uint64_t target = next_listen_num_;
if (!indicator_->cv.wait_for(
lck, std::chrono::milliseconds(timeout_ms), [target, this]() {
return this->indicator_->written_info_num > target ||
this->is_shutdown_.load();
})) {
ADEBUG << "timeout";
return false;
}
if (is_shutdown_.load()) {
AINFO << "notifier is shutdown.";
return false;
}
}
if (next_listen_num_ == 0) {
next_listen_num_ = indicator_->written_info_num - 1;
}
auto idx = next_listen_num_ % kBufLength;
*info = indicator_->infos[idx];
next_listen_num_ += 1;
return true;
}
- 在
Listen返回true後，ShmDispatcher的inner thread就會開始讀readable_info，找出對應Segment，把數據deserialize為object
//cyber/transport/dispatcher/shm_dispatcher.cc
void ShmDispatcher::ReadMessage(uint64_t channel_id, uint32_t block_index) {
ADEBUG << "Reading sharedmem message: "
<< GlobalData::GetChannelById(channel_id)
<< " from block: " << block_index;
auto rb = std::make_shared<ReadableBlock>();
rb->index = block_index;
if (!segments_[channel_id]->AcquireBlockToRead(rb.get())) {
AWARN << "fail to acquire block, channel: "
<< GlobalData::GetChannelById(channel_id)
<< " index: " << block_index;
return;
}
MessageInfo msg_info;
const char* msg_info_addr =
reinterpret_cast<char*>(rb->buf) + rb->block->msg_size();
if (msg_info.DeserializeFrom(msg_info_addr, rb->block->msg_info_size())) {
OnMessage(channel_id, rb, msg_info);
} else {
AERROR << "error msg info of channel:"
<< GlobalData::GetChannelById(channel_id);
}
segments_[channel_id]->ReleaseReadBlock(*rb);
}
void ShmDispatcher::OnMessage(uint64_t channel_id,
const std::shared_ptr<ReadableBlock>& rb,
const MessageInfo& msg_info) {
if (is_shutdown_.load()) {
return;
}
ListenerHandlerBasePtr* handler_base = nullptr;
if (msg_listeners_.Get(channel_id, &handler_base)) {
auto handler = std::dynamic_pointer_cast<ListenerHandler<ReadableBlock>>(
*handler_base);
handler->Run(rb, msg_info);
} else {
AERROR << "Cant find " << GlobalData::GetChannelById(channel_id)
<< "'s handler.";
}
}
void ShmDispatcher::ThreadFunc() {
ReadableInfo readable_info;
while (!is_shutdown_.load()) {
if (!notifier_->Listen(100, &readable_info)) {
ADEBUG << "listen failed.";
continue;
}
uint64_t host_id = readable_info.host_id();
if (host_id != host_id_) {
ADEBUG << "shm readable info from other host.";
continue;
}
uint64_t channel_id = readable_info.channel_id();
uint32_t block_index = readable_info.block_index();
{
ReadLockGuard<AtomicRWLock> lock(segments_lock_);
if (segments_.count(channel_id) == 0) {
continue;
}
// check block index
if (previous_indexes_.count(channel_id) == 0) {
previous_indexes_[channel_id] = UINT32_MAX;
}
uint32_t& previous_index = previous_indexes_[channel_id];
if (block_index != 0 && previous_index != UINT32_MAX) {
if (block_index == previous_index) {
ADEBUG << "Receive SAME index " << block_index << " of channel "
<< channel_id;
} else if (block_index < previous_index) {
ADEBUG << "Receive PREVIOUS message. last: " << previous_index
<< ", now: " << block_index;
} else if (block_index - previous_index > 1) {
ADEBUG << "Receive JUMP message. last: " << previous_index
<< ", now: " << block_index;
}
}
previous_index = block_index;
ReadMessage(channel_id, block_index);
}
}
}
- 最後由
ListenHandler找出要調用那幾個回調並調用
// cyber/transport/message/listener_handler.h
template <typename MessageT>
void ListenerHandler<MessageT>::Run(const Message& msg,
const MessageInfo& msg_info) {
signal_(msg, msg_info);
uint64_t oppo_id = msg_info.sender_id().HashValue();
ReadLockGuard<AtomicRWLock> lock(rw_lock_);
if (signals_.find(oppo_id) == signals_.end()) {
return;
}
(*signals_[oppo_id])(msg, msg_info);
}
//cyber/base/signal.h
void operator()(Args... args) {
SlotList local;
{
std::lock_guard<std::mutex> lock(mutex_);
for (auto& slot : slots_) {
local.emplace_back(slot);
}
}
if (!local.empty()) {
for (auto& slot : local) {
(*slot)(args...);
}
}
ClearDisconnectedSlots();
}
shared memory的syscall
最後記錄一下會用到的一些shared memory相關的syscall
- 指定id,大小，生成一塊shared memory
shmid = shmget(id_, conf_.managed_shm_size(), 0644 | IPC_CREAT | IPC_EXCL);
- 指到那塊shared memory的memory location
managed_shm_ = shmat(shmid, nullptr, 0);
- 用inplace new去指定用那塊內存生成object
state_ = new (managed_shm_) State(conf_.ceiling_msg_size());
4. 清理shared memory
shmdt(managed_shm_);
shmctl(shmid, IPC_RMID, 0);
5.這不是直接的syscall。使mutex,conditional_variable在共享內存中可以跨進程工作
pthread_mutexattr_t mtx_attr;
pthread_mutexattr_init(&mtx_attr);
pthread_mutexattr_setpshared(&mtx_attr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(indicator_->mtx.native_handle(), &mtx_attr);
pthread_condattr_t cond_attr;
pthread_condattr_init(&cond_attr);
pthread_condattr_setpshared(&cond_attr, PTHREAD_PROCESS_SHARED);
pthread_cond_init(indicator_->cv.native_handle(), &cond_attr);