TCP三步握手建立连接(1)-----主动连接syn包发送

TCP连接的建立一般是服务器端进入监听状态，然后客户端调用connect来连接。本文分析connect调用的过程，也即第一个syn报文的发送。

 

connect对应的系统调用是sys_connect

 SYSCALL_DEFINE3(connect, int, fd, struct sockaddr __user *, uservaddr,
         int, addrlen)
 {
     struct socket *sock;
     struct sockaddr_storage address;
     int err, fput_needed;
         /* 找到文件描述符对应的BSD socket结构，在前面的socket调用中建立*/
     sock = sockfd_lookup_light(fd, &err, &fput_needed);
     if (!sock)
         goto out;
         /* copy对端的地址到内核空间 */
     err = move_addr_to_kernel(uservaddr, addrlen, (struct sockaddr *)&address);
     if (err < 0)
         goto out_put;

     err =
         security_socket_connect(sock, (struct sockaddr *)&address, addrlen);
     if (err)
         goto out_put;
         /* 调用该BSD socket对应的connect调用 */
     err = sock->ops->connect(sock, (struct sockaddr *)&address, addrlen,
                  sock->file->f_flags);
 out_put:
         /* 释放文件的引用 */
     fput_light(sock->file, fput_needed);
 out:
     return err;

 

上面函数中的sock是在前面的socket调用的时候就初始化的，对应于TCP的BSD socket 的操作符集是inet_stream_ops

注：网络子模块在内核初始化的时候就注册了TCP，UDP和RAW3中协议，具体在af_inet.c中的inet_init函数中。

 onst struct proto_ops inet_stream_ops = {
     .family        = PF_INET,
     .owner         = THIS_MODULE,
     .release       = inet_release,
     .bind          = inet_bind,
     .connect       = inet_stream_connect,
     .socketpair    = sock_no_socketpair,
     .accept        = inet_accept,
     .getname       = inet_getname,
     .poll          = tcp_poll,
     .ioctl         = inet_ioctl,
     .listen        = inet_listen,
     .shutdown      = inet_shutdown,
     .setsockopt    = sock_common_setsockopt,
     .getsockopt    = sock_common_getsockopt,
     .sendmsg       = tcp_sendmsg,
     .recvmsg       = sock_common_recvmsg,
     .mmap          = sock_no_mmap,
     .sendpage      = tcp_sendpage,
     .splice_read       = tcp_splice_read,
 #ifdef CONFIG_COMPAT
     .compat_setsockopt = compat_sock_common_setsockopt,
     .compat_getsockopt = compat_sock_common_getsockopt,
 #endif
 };

从上面的结构中可以看到connect对应的函数是inet_stream_connect。下面分析该函数

 /*
  *  Connect to a remote host. There is regrettably still a little
  *  TCP 'magic' in here.
  */
 int inet_stream_connect(struct socket *sock, struct sockaddr *uaddr,
             int addr_len, int flags)
 {
     struct sock *sk = sock->sk;
     int err;
     long timeo;

     lock_sock(sk);

     if (uaddr->sa_family == AF_UNSPEC) {
         err = sk->sk_prot->disconnect(sk, flags);
         sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED;
         goto out;
     }

     switch (sock->state) {
     default:
         err = -EINVAL;
         goto out;
     case SS_CONNECTED:     /* 该BSD socket已连接*/
         err = -EISCONN;
         goto out;
     case SS_CONNECTING:   /* 该BSD socket正在连接*/
         err = -EALREADY;
         /* Fall out of switch with err, set for this state */
         break;
     case SS_UNCONNECTED:
         err = -EISCONN;
         if (sk->sk_state != TCP_CLOSE)
             goto out;
                 /* INET SOCKET 调用协议特有connect操作符 */
         err = sk->sk_prot->connect(sk, uaddr, addr_len);
         if (err < 0)
             goto out;
                 /* 上面的调用完成后，连接并没有完成，*/
         sock->state = SS_CONNECTING;

         /* Just entered SS_CONNECTING state; the only
          * difference is that return value in non-blocking
          * case is EINPROGRESS, rather than EALREADY.
          */
         err = -EINPROGRESS;
         break;
     }
         /* 获取连接超时时间*/
     timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);

     if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
         /* Error code is set above 进入定时等待 */
         if (!timeo || !inet_wait_for_connect(sk, timeo))
             goto out;

         err = sock_intr_errno(timeo);
         if (signal_pending(current))
             goto out;
     }

     /* Connection was closed by RST, timeout, ICMP error
      * or another process disconnected us.
      */
     if (sk->sk_state == TCP_CLOSE)
         goto sock_error;

     /* sk->sk_err may be not zero now, if RECVERR was ordered by user
      * and error was received after socket entered established state.
      * Hence, it is handled normally after connect() return successfully.
      */

     sock->state = SS_CONNECTED;
     err = 0;
 out:
     release_sock(sk);
     return err;

 sock_error:
     err = sock_error(sk) ? : -ECONNABORTED;
     sock->state = SS_UNCONNECTED;
     if (sk->sk_prot->disconnect(sk, flags))
         sock->state = SS_DISCONNECTING;
     goto out;
 }

 

对于INET socket中的tcp连接，协议特有操作符集为tcp_prot

 struct proto tcp_prot = {
     .name           = "TCP",
     .owner          = THIS_MODULE,
     .close          = tcp_close,
     .connect        = tcp_v4_connect,
     .disconnect     = tcp_disconnect,
     .accept         = inet_csk_accept,
     .ioctl          = tcp_ioctl,
     .init           = tcp_v4_init_sock,
     .destroy        = tcp_v4_destroy_sock,
     .shutdown       = tcp_shutdown,
     .setsockopt     = tcp_setsockopt,
     .getsockopt     = tcp_getsockopt,
     .recvmsg        = tcp_recvmsg,
     .backlog_rcv        = tcp_v4_do_rcv,
     .hash           = inet_hash,
     .unhash         = inet_unhash,
     .get_port       = inet_csk_get_port,
     .enter_memory_pressure  = tcp_enter_memory_pressure,
     .sockets_allocated  = &tcp_sockets_allocated,
     .orphan_count       = &tcp_orphan_count,
     .memory_allocated   = &tcp_memory_allocated,
     .memory_pressure    = &tcp_memory_pressure,
     .sysctl_mem     = sysctl_tcp_mem,
     .sysctl_wmem        = sysctl_tcp_wmem,
     .sysctl_rmem        = sysctl_tcp_rmem,
     .max_header     = MAX_TCP_HEADER,
     .obj_size       = sizeof(struct tcp_sock),
     .slab_flags     = SLAB_DESTROY_BY_RCU,
     .twsk_prot      = &tcp_timewait_sock_ops,
     .rsk_prot       = &tcp_request_sock_ops,
     .h.hashinfo     = &tcp_hashinfo,
 #ifdef CONFIG_COMPAT
     .compat_setsockopt  = compat_tcp_setsockopt,
     .compat_getsockopt  = compat_tcp_getsockopt,
 #endif
 };

 

可以看出这2个操作符集中好多函数是相同的，不过它们是出于不同层次中的，从系统调用的角度来看，BSD socket的操作符集是先

被调用的，然后再调用对应的INET socket的操作符集。

对于TCP，流程进入tcp_v4_connect函数

 /* This will initiate an outgoing connection. */
 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 {
     struct inet_sock *inet = inet_sk(sk);
     struct tcp_sock *tp = tcp_sk(sk);
     struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
     struct rtable *rt;
     __be32 daddr, nexthop;
     int tmp;
     int err;

     if (addr_len < sizeof(struct sockaddr_in))
         return -EINVAL;

     if (usin->sin_family != AF_INET)
         return -EAFNOSUPPORT;
         /* 开始准备路由 */
     nexthop = daddr = usin->sin_addr.s_addr;
     if (inet->opt && inet->opt->srr) {
         if (!daddr)
             return -EINVAL;
         nexthop = inet->opt->faddr;
     }
         /* 调用路由模块获取出口信息，这里不深入 */
     tmp = ip_route_connect(&rt, nexthop, inet->saddr,
                    RT_CONN_FLAGS(sk), sk->sk_bound_dev_if,
                    IPPROTO_TCP,
                    inet->sport, usin->sin_port, sk, 1);
     if (tmp < 0) {
         if (tmp == -ENETUNREACH)
             IP_INC_STATS_BH(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
         return tmp;
     }
         /* 如果获取的路由是广播或多播域， 返回网络不可达，tcp不支持多播与广播 */
     if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
         ip_rt_put(rt);
         return -ENETUNREACH;
     }

     if (!inet->opt || !inet->opt->srr)
         daddr = rt->rt_dst;

     if (!inet->saddr)
         inet->saddr = rt->rt_src;
     inet->rcv_saddr = inet->saddr;

     if (tp->rx_opt.ts_recent_stamp && inet->daddr != daddr) {
         /* Reset inherited state */
         tp->rx_opt.ts_recent    = 0;
         tp->rx_opt.ts_recent_stamp = 0;
         tp->write_seq           = 0;
     }

     if (tcp_death_row.sysctl_tw_recycle &&
         !tp->rx_opt.ts_recent_stamp && rt->rt_dst == daddr) {
         struct inet_peer *peer = rt_get_peer(rt);
         /*
          * VJ's idea. We save last timestamp seen from
          * the destination in peer table, when entering state
          * TIME-WAIT * and initialize rx_opt.ts_recent from it,
          * when trying new connection.
          */
         if (peer != NULL &&
             peer->tcp_ts_stamp + TCP_PAWS_MSL >= get_seconds()) {
             tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
             tp->rx_opt.ts_recent = peer->tcp_ts;
         }
     }

     inet->dport = usin->sin_port;
     inet->daddr = daddr;

     inet_csk(sk)->icsk_ext_hdr_len = 0;
     if (inet->opt)
         inet_csk(sk)->icsk_ext_hdr_len = inet->opt->optlen;
         /* mss_clamp */
     tp->rx_opt.mss_clamp = 536;

     /* Socket identity is still unknown (sport may be zero).
      * However we set state to SYN-SENT and not releasing socket
      * lock select source port, enter ourselves into the hash tables and
      * complete initialization after this.
      */
     tcp_set_state(sk, TCP_SYN_SENT);
     err = inet_hash_connect(&tcp_death_row, sk);
     if (err)
         goto failure;

     err = ip_route_newports(&rt, IPPROTO_TCP,
                 inet->sport, inet->dport, sk);
     if (err)
         goto failure;

     /* OK, now commit destination to socket.  */
     sk->sk_gso_type = SKB_GSO_TCPV4;
     sk_setup_caps(sk, &rt->u.dst);

     if (!tp->write_seq)
         tp->write_seq = secure_tcp_sequence_number(inet->saddr,
                                inet->daddr,
                                inet->sport,
                                usin->sin_port);
         /* id是IP包头的id域 */
     inet->id = tp->write_seq ^ jiffies;

     err = tcp_connect(sk);
     rt = NULL;
     if (err)
         goto failure;

     return 0;

 failure:
     /*
      * This unhashes the socket and releases the local port,
      * if necessary.
      */
     tcp_set_state(sk, TCP_CLOSE);
     ip_rt_put(rt);
     sk->sk_route_caps = 0;
     inet->dport = 0;
     return err;
 }

该函数的主要功能是准备好路由，如果源端口没有指定，还要选择一个端口，然后再次更新路由信息；代表该连接的sk结构加入

到对应的hash表中（已连接ehash）。获取一个write_seq，以及当 sysctl_tw_recycle设置时，读取上次连接（如果存在）

进入time-wait时保存的时戳，赋给当前连接的rx_opt结构中。

 

最后该函数调用tcp_connect来完成连接

 /*
  * Build a SYN and send it off.
  */
 int tcp_connect(struct sock *sk)
 {
     struct tcp_sock *tp = tcp_sk(sk);
     struct sk_buff *buff;
         /* 初始化连接对应的INET socket结构的参数，为连接做准备 */
     tcp_connect_init(sk);
         /* 获取一个skb，由于是syn包，没有数据，所以大小是MAX_TCP_HEADER的16位对齐 */
     buff = alloc_skb_fclone(MAX_TCP_HEADER + 15, sk->sk_allocation);
     if (unlikely(buff == NULL))
         return -ENOBUFS;

     /* Reserve space for headers. */
     skb_reserve(buff, MAX_TCP_HEADER);

     tp->snd_nxt = tp->write_seq;
         /* 设置skb相关参数 */
     tcp_init_nondata_skb(buff, tp->write_seq++, TCPCB_FLAG_SYN);
         /* 设置ECN */
     TCP_ECN_send_syn(sk, buff);

     /* Send it off. */
         /* 保存该数据包的发送时间*/
     TCP_SKB_CB(buff)->when = tcp_time_stamp;
     tp->retrans_stamp = TCP_SKB_CB(buff)->when;
     skb_header_release(buff);
         /* 加入发送队列，待确认后在丢弃*/
     __tcp_add_write_queue_tail(sk, buff);
     sk->sk_wmem_queued += buff->truesize;
     sk_mem_charge(sk, buff->truesize);
     tp->packets_out += tcp_skb_pcount(buff);
     tcp_transmit_skb(sk, buff, 1, GFP_KERNEL);

     /* We change tp->snd_nxt after the tcp_transmit_skb() call
      * in order to make this packet get counted in tcpOutSegs.
      */
     tp->snd_nxt = tp->write_seq;
     tp->pushed_seq = tp->write_seq;
     TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);

     /* Timer for repeating the SYN until an answer. */
     inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
                   inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
     return 0;
 }

 

该函数的流程是

1. 初始化该次连接的sk结构
2. 分配一个skb数据包
3. 初始化skb
4. skb加入发送队列后，调用tcp_transmit_skb发送该数据包
5. 更新snd_nxt，并启动超时定时器。

首先来看sk结构的初始化，也就是tcp_connect_init函数

 /*
  * Do all connect socket setups that can be done AF independent.
  */
 static void tcp_connect_init(struct sock *sk)
 {
     struct dst_entry *dst = __sk_dst_get(sk);
     struct tcp_sock *tp = tcp_sk(sk);
     __u8 rcv_wscale;

     /* We'll fix this up when we get a response from the other end.
      * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
      */
     tp->tcp_header_len = sizeof(struct tcphdr) +
         (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);

 #ifdef CONFIG_TCP_MD5SIG
     if (tp->af_specific->md5_lookup(sk, sk) != NULL)
         tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
 #endif

     /* If user gave his TCP_MAXSEG, record it to clamp */
     if (tp->rx_opt.user_mss)
         tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
     tp->max_window = 0;
         /* 初始化MTU probe*/
     tcp_mtup_init(sk);
         /* 设置mss */
     tcp_sync_mss(sk, dst_mtu(dst));

     if (!tp->window_clamp)
         tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
     tp->advmss = dst_metric(dst, RTAX_ADVMSS);
     if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss)
         tp->advmss = tp->rx_opt.user_mss;

     tcp_initialize_rcv_mss(sk);
         /* 根据接收空间大小初始化一个通告窗口 */
     tcp_select_initial_window(tcp_full_space(sk),
                   tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
                   &tp->rcv_wnd,
                   &tp->window_clamp,
                   sysctl_tcp_window_scaling,
                   &rcv_wscale);

     tp->rx_opt.rcv_wscale = rcv_wscale;
     tp->rcv_ssthresh = tp->rcv_wnd;

     sk->sk_err = 0;
     sock_reset_flag(sk, SOCK_DONE);
     tp->snd_wnd = 0;
         /* 更新一些滑动窗口的成员*/
     tcp_init_wl(tp, tp->write_seq, 0);
     tp->snd_una = tp->write_seq;
     tp->snd_sml = tp->write_seq;
     tp->snd_up = tp->write_seq;
     tp->rcv_nxt = 0;
     tp->rcv_wup = 0;
     tp->copied_seq = 0;

     inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
     inet_csk(sk)->icsk_retransmits = 0;
     tcp_clear_retrans(tp);
 }

 

接下来发送数据包，调用tcp_transimit_skb函数

 /* This routine actually transmits TCP packets queued in by
  * tcp_do_sendmsg().  This is used by both the initial
  * transmission and possible later retransmissions.
  * All SKB's seen here are completely headerless.  It is our
  * job to build the TCP header, and pass the packet down to
  * IP so it can do the same plus pass the packet off to the
  * device.
  * 该函数是TCP模块通用的发送函数，在此要负责组装TCP头部
  * We are working here with either a clone of the original
  * SKB, or a fresh unique copy made by the retransmit engine.
  */

 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
                 gfp_t gfp_mask)
 {
     const struct inet_connection_sock *icsk = inet_csk(sk);
     struct inet_sock *inet;
     struct tcp_sock *tp;
     struct tcp_skb_cb *tcb;
     struct tcp_out_options opts;
     unsigned tcp_options_size, tcp_header_size;
     struct tcp_md5sig_key *md5;
     __u8 *md5_hash_location;
     struct tcphdr *th;
     int err;

     BUG_ON(!skb || !tcp_skb_pcount(skb));

     /* If congestion control is doing timestamping, we must
      * take such a timestamp before we potentially clone/copy.
      */
     if (icsk->icsk_ca_ops->flags & TCP_CONG_RTT_STAMP)
         __net_timestamp(skb);
         /* 因为发送时要保留skb以备重传，所以大部分时候都设置了clone_it，不过发送ack 除外*/
     if (likely(clone_it)) {
         if (unlikely(skb_cloned(skb)))
             skb = pskb_copy(skb, gfp_mask);
         else
             skb = skb_clone(skb, gfp_mask);
         if (unlikely(!skb))
             return -ENOBUFS;
     }

     inet = inet_sk(sk);
     tp = tcp_sk(sk);
     tcb = TCP_SKB_CB(skb);
     memset(&opts, 0, sizeof(opts));
         /* 获取tcp选项部分信息，分为syn包和普通包2部分，因为有的选项只在syn中设置*/
     if (unlikely(tcb->flags & TCPCB_FLAG_SYN))
         tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
     else
         tcp_options_size = tcp_established_options(sk, skb, &opts,
                                &md5);
     tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
         /* 该函数判断是否网络中有数据存在，没有就通知拥塞控制模块 */
     if (tcp_packets_in_flight(tp) == 0)
         tcp_ca_event(sk, CA_EVENT_TX_START);

     skb_push(skb, tcp_header_size);
     skb_reset_transport_header(skb);
     skb_set_owner_w(skb, sk);

     /* Build TCP header and checksum it. */
     th = tcp_hdr(skb);
     th->source       = inet->sport;
     th->dest     = inet->dport;
     th->seq          = htonl(tcb->seq);
     th->ack_seq      = htonl(tp->rcv_nxt);
     *(((__be16 *)th) + 6)   = htons(((tcp_header_size >> 2) << 12) |
                     tcb->flags);

     if (unlikely(tcb->flags & TCPCB_FLAG_SYN)) {
         /* RFC1323: The window in SYN & SYN/ACK segments
          * is never scaled.SYN包和SYN/ACK包中窗口不扩放
          */
         th->window   = htons(min(tp->rcv_wnd, 65535U));
     } else {
         th->window   = htons(tcp_select_window(sk));
     }
     th->check        = 0;
     th->urg_ptr      = 0;

     /* The urg_mode check is necessary during a below snd_una win probe */
     if (unlikely(tcp_urg_mode(tp) &&
              between(tp->snd_up, tcb->seq + 1, tcb->seq + 0xFFFF))) {
         th->urg_ptr      = htons(tp->snd_up - tcb->seq);
         th->urg          = 1;
     }
         /* 填充TCP 选项字段 */
     tcp_options_write((__be32 *)(th + 1), tp, &opts, &md5_hash_location);
     if (likely((tcb->flags & TCPCB_FLAG_SYN) == 0))
                 /* 如果不是SYN包的话，就检查是否需要向对方发送ECN */
         TCP_ECN_send(sk, skb, tcp_header_size);

 #ifdef CONFIG_TCP_MD5SIG
     /* Calculate the MD5 hash, as we have all we need now */
     if (md5) {
         sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
         tp->af_specific->calc_md5_hash(md5_hash_location,
                            md5, sk, NULL, skb);
     }
 #endif
         /* 校验和相关计算*/
     icsk->icsk_af_ops->send_check(sk, skb->len, skb);

     if (likely(tcb->flags & TCPCB_FLAG_ACK))
                 /* 如果发送了ACK的话，就。。。*/
         tcp_event_ack_sent(sk, tcp_skb_pcount(skb));

     if (skb->len != tcp_header_size)
                  /* 如果发送了数据的话，就。。。*/
         tcp_event_data_sent(tp, skb, sk);

     if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
         TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
         /* 调用IP层的发送函数 */
     err = icsk->icsk_af_ops->queue_xmit(skb, 0);
     if (likely(err <= 0))
         return err;

     tcp_enter_cwr(sk, 1);

     return net_xmit_eval(err);
 }

 

skb发送后，connect并没有返回，因为此时连接还没有建立，tcp进入等待状态，此时回到前面的inet_stream_connect函数

在发送syn后进入等待状态

 static long inet_wait_for_connect(struct sock *sk, long timeo)
 {
     DEFINE_WAIT(wait);
         /* sk_sleep 保存此INET SOCKET的等待队列 */
     prepare_to_wait(sk->sk_sleep, &wait, TASK_INTERRUPTIBLE);

     /* Basic assumption: if someone sets sk->sk_err, he _must_
      * change state of the socket from TCP_SYN_*.
      * Connect() does not allow to get error notifications
      * without closing the socket.
      */
         /* 定时等待知道状态变化 */
     while ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
         release_sock(sk);
         timeo = schedule_timeout(timeo);
         lock_sock(sk);
         if (signal_pending(current) || !timeo)
             break;
         prepare_to_wait(sk->sk_sleep, &wait, TASK_INTERRUPTIBLE);
     }
     finish_wait(sk->sk_sleep, &wait);
     return timeo;
 }

 

下文中将分析第二步：服务器端返回syn_ack包