rgw && message

librados::IoCtxImpl::operate_read()                    //do read operation
    Objecter：md_config_obs_t：Dispatcher
        prepare_read_op()                              //new a Op instance is used by Objecter
        op_submit()                                    //submit operation
            _op_submit_with_budget()
                _op_submit()
                    _cal_target()                       //1.cacluate th up/active OSD sets 2.select primary osd as the target
                    _get_session()                      //1.Objecter keeps an OSDSession with each osd 2.new MOSDOp which will be used by OSD
                    _prepare_osd_osd_op()
                    _session_op_assign()                
                    _send_op()
                        op->session-con->send_message()
                            SimpleMessager->send_message()
                                _send_message()
                                    submit_message()
                                        Pip->_send(m)
                                            Pip->cond.signal()
                                                |
                                                |
                                            Pipe::write()
                                                _get_next_outgoing()
                                                write_message()
                                                    do_sendmsg()
                                                        sendmsg()


OSDSession *s = new OSDSession(cct, osd);
  osd_sessions[osd] = s;
  s->con = messenger->get_connection(osdmap->get_inst(osd));
  logger->inc(l_osdc_osd_session_open);
  logger->inc(l_osdc_osd_sessions, osd_sessions.size());
  s->get();
  *session = s;

using SimpleMessager
SimpleMessenger::get_connection()
    return local_connection  //if my_instd.addr equal dest.addr
    _lookup_pipe(dest.addr)  //if don't found exsit pipe we will new a pipe instance


 // create pipe
  Pipe *pipe = new Pipe(this, Pipe::STATE_CONNECTING,
            static_cast<PipeConnection*>(con));
  pipe->pipe_lock.Lock();
  pipe->set_peer_type(type);
  pipe->set_peer_addr(addr);
  pipe->policy = get_policy(type);
  pipe->start_writer();
  if (first)
    pipe->_send(first);
  pipe->pipe_lock.Unlock();
  pipe->register_pipe();                         //add pip instance to SimpleMessager->rack_pipe map
  pipes.insert(pipe);                            //add pip instance to SimpleMessager->pipes set

  return pipe;

 client side commponents Messager
 Responsibility
    a network entity in ceph
    handles transmission and reception osd messages
    based on socket
 Implementation
    SimpleMessenger
    AsyncMessenger
    Xio
 Role
    cluster_messenger:responsible for communicationg with other OSDs/Monitors
    client_messenger: responsible for communicating with client(qemu/kvm、ceph-fuse，etc)
    hbclient_messenger、hb_front_server_messenger hb_back_server_messenger



 SimpleMessager:

/* * This class handles transmission and reception of messages. Generally * speaking, there are several major components: * * - Connection(PipConnection) * Each logical session is associated with a Connection. * - Pipe * Each network connection is handled through a pipe, which handles * the input and output of each message. There is normally a 1:1 * relationship between Pipe and Connection, but logical sessions may * get handed off between Pipes when sockets reconnect or during * connection races. * - IncomingQueue * Incoming messages are associated with an IncomingQueue, and there * is one such queue associated with each Pipe. * - DispatchQueue * IncomingQueues get queued in the DIspatchQueue, which is responsible * for doing a round-robin sweep and processing them via a worker thread. * - SimpleMessenger * It's the exterior class passed to the external message handler and * most of the API details. * * Lock ordering: * * SimpleMessenger::lock * Pipe::pipe_lock * DispatchQueue::lock * IncomingQueue::lock */

 Accepter:
 /** * If the SimpleMessenger binds to a specific address, the Accepter runs * and listens for incoming connections. */


 DispatchQueue:
 /** * The DispatchQueue contains all the Pipes which have Messages * they want to be dispatched, carefully organized by Message priority * and permitted to deliver in a round-robin fashion. * See SimpleMessenger::dispatch_entry for details. */

 DispatchThread (in DispatchQueue)
  /** * The DispatchThread runs dispatch_entry to empty out the dispatch_queue. */


 Pip
 /** * The Pipe is the most complex SimpleMessenger component. It gets * two threads, one each for reading and writing on a socket it's handed * at creation time, and is responsible for everything that happens on * that socket. Besides message transmission, it's responsible for * propagating socket errors to the SimpleMessenger and then sticking * around in a state where it can provide enough data for the SimpleMessenger * to provide reliable Message delivery when it manages to reconnect. */
   Reader(in Pip)
   /** * The Reader thread handles all reads off the socket -- not just * Messages, but also acks and other protocol bits (excepting startup, * when the Writer does a couple of reads). * All the work is implemented in Pipe itself, of course. */
     Writer(in Pip)
    /** * The Writer thread handles all writes to the socket (after startup). * All the work is implemented in Pipe itself, of course. */