LWIP学习笔记（5）ARP协议

etharp.c/h实现了ARP协议全部数据结构和函数

ARP数据结构

ARP表
lwip中描述arp表的数据结构叫etharp_entry,如下
etharp.c中

etharp.h中


当ip层发送数据包时，会在arp找目的ip对应的mac地址，如果找不到，arp会针对此ip创建一个表项，并发送一个arp请求，同时把要发的数据包暂时放到缓冲队列指针上，等到收到arp应答知道目标ip对应的mac后再报缓冲队列指针上的pbuf发送出去。

struct etharp_entry数据结构可用下图表示

上图表示ip层有某个目的ip上有2个pbuf要发送出去，但是目前还不知道mac，就把这2个pbuf挂在缓冲队列指针上。

state指明该表的状态，在etharp.c中

有4种状态

• ETHARP_STATE_EMPTY
  没有记录任何信息
• ETHARP_STATE_PENDING
  表示只记录了ip但是没有记录mac，可能是发送了arp请求，但是还没收到应答
• ETHARP_STATE_STABLE
  表示记录了一对完整的ip和mac
• ETHARP_STATE_STABLE_REREQUESTING
  在ETHARP_STATE_STABLE状态后，定期发送arp请求以更新arp表，当发送后就由ETHARP_STATE_STABLE变成ETHARP_STATE_STABLE_REREQUESTING ，在收到arp应答完成更新后，又变成ETHARP_STATE_STABLE
  etharp.c中定义了arp表
  
  opt.h
  
  所以默认维护10张arp表
  ctime指明该表的存在时间，在etharp.c中内核每5s会调用etharp_tmr，根据时间删除那些旧的不用的arp表以提高使用率

ARP报文

arp报文数据结构在etharp.h定义
首先是以太网首部

arp数据结构

发送一个arp报文调用如下函数

这个函数里面是申请一个pbuf，然后根据输入参数填充pbuf，最后调用netif->linkoutput即low_level_output函数发送数据，最后释放pbuf返回。
而arp请求函数是对上面函数的调用并填上输入参数

ARP输入

ARP的输入是从ethernetif_input函数输入，这就是之前网络接口注册的数据包接收函数，他是调用low_level_input从网卡接收数据，在提交给上层使用，在ethernetif.c中

ethernet_input在etharp.c中

err_t ethernet_input(struct pbuf *p, struct netif *netif)
{
  struct eth_hdr* ethhdr;
  u16_t type;

  if (p->len <= SIZEOF_ETH_HDR) {//14以太网首部
    /* a packet with only an ethernet header (or less) is not valid for us */
    goto free_and_return;
  }

  /* points to packet payload, which starts with an Ethernet header */
  ethhdr = (struct eth_hdr *)p->payload;

  type = ethhdr->type;

  switch (type) {
      case PP_HTONS(ETHTYPE_IP):
#if ETHARP_TRUST_IP_MAC
      /* update ARP table */
      etharp_ip_input(netif, p);
#endif /* ETHARP_TRUST_IP_MAC */
      /* skip Ethernet header */
      if(pbuf_header(p, -ip_hdr_offset)) {
        LWIP_ASSERT("Can't move over header in packet", 0);
        goto free_and_return;
      } else {
        /* pass to IP layer */
        ip_input(p, netif);
      }
      break;
      
    case PP_HTONS(ETHTYPE_ARP):
      /* pass p to ARP module */
      etharp_arp_input(netif, (struct eth_addr*)(netif->hwaddr), p);
      break;

    default:
      goto free_and_return;
  }

  /* This means the pbuf is freed or consumed,
     so the caller doesn't have to free it again */
  return ERR_OK;

free_and_return:
  pbuf_free(p);
  return ERR_OK;
}

上面大概意思是从网卡接收数据后如果是

• ETHTYPE_IP类型
  调用 etharp_ip_input(netif, p);使用ip头部和以太网头部更新arp表并调用ip_input(p, netif);交给上层ip处理
• ETHTYPE_ARP类型
  调用 etharp_arp_input(netif, (struct eth_addr*)(netif->hwaddr), p);交给arp处理

etharp_arp_input做如下处理

• 如果收到arp应答包
  说明这是本机之前发出arp请求的响应，则根据包中数据更新arp表
• 如果收到arp请求包，但是请求ip和本机不匹配
  可以将这个ip加入arp表，更新arp表
• 如果收到arp请求包，并且请求ip和本机匹配
  更新arp表，并且要返回一个arp应答
  在etharp.c中

/**
 * Responds to ARP requests to us. Upon ARP replies to us, add entry to cache  
 * send out queued IP packets. Updates cache with snooped address pairs.
 *
 * Should be called for incoming ARP packets. The pbuf in the argument
 * is freed by this function.
 *
 * @param netif The lwIP network interface on which the ARP packet pbuf arrived.
 * @param ethaddr Ethernet address of netif.
 * @param p The ARP packet that arrived on netif. Is freed by this function.
 *
 * @return NULL
 *
 * @see pbuf_free()
 */
static void
etharp_arp_input(struct netif *netif, struct eth_addr *ethaddr, struct pbuf *p)
{
  struct etharp_hdr *hdr;
  struct eth_hdr *ethhdr;
  /* these are aligned properly, whereas the ARP header fields might not be */
  ip_addr_t sipaddr, dipaddr;//暂存arp中源目ip
  u8_t for_us;

  /* drop short ARP packets: we have to check for p->len instead of p->tot_len here
     since a struct etharp_hdr is pointed to p->payload, so it musn't be chained! */
  if (p->len < SIZEOF_ETHARP_PACKET) {//14+28,以太网首部和arp报文要在一个pbuf内
    pbuf_free(p);
    return;
  }

  ethhdr = (struct eth_hdr *)p->payload;//指向以太网首部
  hdr = (struct etharp_hdr *)((u8_t*)ethhdr + SIZEOF_ETH_HDR);//指向arp报文

  /* RFC 826 "Packet Reception": *///ARP包合法性判断
  if ((hdr->hwtype != PP_HTONS(HWTYPE_ETHERNET)) ||//以太网类型
      (hdr->hwlen != ETHARP_HWADDR_LEN) ||//硬件地址长度
      (hdr->protolen != sizeof(ip_addr_t)) ||//协议类型长度
      (hdr->proto != PP_HTONS(ETHTYPE_IP)))  {//协议类型位ip
    pbuf_free(p);
    return;
  }
	//注意这里不是字节对齐的
  /* Copy struct ip_addr2 to aligned ip_addr, to support compilers without
   * structure packing (not using structure copy which breaks strict-aliasing rules). */
  IPADDR2_COPY(&sipaddr, &hdr->sipaddr);//copy发送方ip
  IPADDR2_COPY(&dipaddr, &hdr->dipaddr);//copy接受方ip
  //这里判断接收方ip是否是本机，用for_us标记
  /* this interface is not configured? */
  if (ip_addr_isany(&netif->ip_addr)) {
    for_us = 0;
  } else {
    /* ARP packet directed to us? */
    for_us = (u8_t)ip_addr_cmp(&dipaddr, &(netif->ip_addr));
  }

  /* ARP message directed to us? 如果是给本机的(请求或响应)，则更新arp表，flag = ETHARP_FLAG_TRY_HARD
      -> add IP address in ARP cache; assume requester wants to talk to us,
         can result in directly sending the queued packets for this host.
     ARP message not directed to us? 如果不是给本机的,也更新arp，flag = ETHARP_FLAG_FIND_ONLY
      ->  update the source IP address in the cache, if present */
  etharp_update_arp_entry(netif, &sipaddr, &(hdr->shwaddr),
                   for_us ? ETHARP_FLAG_TRY_HARD : ETHARP_FLAG_FIND_ONLY);
  //下面根据是收到的是请求还是响应判断是否要发送arp应答
  /* now act on the message itself */
  switch (hdr->opcode) {
  /* ARP request? */
  case PP_HTONS(ARP_REQUEST)://arp请求，则需要做arp应答，直接改写接收到的pbuf字段返回
    /* ARP request. If it asked for our address, we send out a
     * reply. In any case, we time-stamp any existing ARP entry,
     * and possiby send out an IP packet that was queued on it. */
    /* ARP request for our address? */
    if (for_us) {
      /* Re-use pbuf to send ARP reply.
         Since we are re-using an existing pbuf, we can't call etharp_raw since
         that would allocate a new pbuf. */
      hdr->opcode = htons(ARP_REPLY);
      IPADDR2_COPY(&hdr->dipaddr, &hdr->sipaddr);
      IPADDR2_COPY(&hdr->sipaddr, &netif->ip_addr);

      ETHADDR16_COPY(&hdr->dhwaddr, &hdr->shwaddr);
      ETHADDR16_COPY(&ethhdr->dest, &hdr->shwaddr);
      ETHADDR16_COPY(&hdr->shwaddr, ethaddr);
      ETHADDR16_COPY(&ethhdr->src, ethaddr);

      /* hwtype, hwaddr_len, proto, protolen and the type in the ethernet header
         are already correct, we tested that before */

      /* return ARP reply */
      netif->linkoutput(netif, p);//发送arp应答
    /* we are not configured? */
    } else if (ip_addr_isany(&netif->ip_addr)) {//虽然是请求但是不是请求本机，所以不做应答
      /* { for_us == 0 and netif->ip_addr.addr == 0 } */
    /* request was not directed to us */
    } else {//
      /* { for_us == 0 and netif->ip_addr.addr != 0 } */
    }
    break;
  case PP_HTONS(ARP_REPLY)://arp应答，不做处理
    /* ARP reply. We already updated the ARP cache earlier. */
    break;
  default:
    break;
  }
  /* free ARP packet */
  pbuf_free(p);
}

上面更新arp表示调用etharp_update_arp_entry(struct netif *netif, ip_addr_t *ipaddr, struct eth_addr *ethaddr, u8_t flags),其内部又调用etharp_find_entry(ip_addr_t *ipaddr, u8_t flags)
etharp_find_entry(ip_addr_t *ipaddr, u8_t flags)负责为ipaddr从那10个arp表中寻找一个合适的表返回给etharp_update_arp_entry，etharp_update_arp_entry再设置这个表项的相应字段并检查是否有要发送的数据包，有则发送出去。

先看etharp_find_entry(ip_addr_t *ipaddr, u8_t flags)
解释下5个局部变量
empty 索引号最低的empty表项
old_stable 年龄最大的stable表项
old_pending 年龄最大且缓冲对列为空的的pending表项
old_queue 年龄最大且缓冲对列不为空的的pending表项

在etharp.c中

/**
 * Search the ARP table for a matching or new entry.
 * 
 * If an IP address is given, return a pending or stable ARP entry that matches
 * the address. If no match is found, create a new entry with this address set,
 * but in state ETHARP_EMPTY. The caller must check and possibly change the
 * state of the returned entry.
 * 
 * If ipaddr is NULL, return a initialized new entry in state ETHARP_EMPTY.
 * 
 * In all cases, attempt to create new entries from an empty entry. If no
 * empty entries are available and ETHARP_FLAG_TRY_HARD flag is set, recycle
 * old entries. Heuristic choose the least important entry for recycling.
 *
 * @param ipaddr IP address to find in ARP cache, or to add if not found.
 * @param flags @see definition of ETHARP_FLAG_*
 * @param netif netif related to this address (used for NETIF_HWADDRHINT)
 *  
 * @return The ARP entry index that matched or is created, ERR_MEM if no
 * entry is found or could be recycled.
 */
static s8_t
etharp_find_entry(ip_addr_t *ipaddr, u8_t flags)
{
  s8_t old_pending = ARP_TABLE_SIZE, old_stable = ARP_TABLE_SIZE;
  s8_t empty = ARP_TABLE_SIZE;
  u8_t i = 0, age_pending = 0, age_stable = 0;//分别记录old_pending和old_stable的年龄
  /* oldest entry with packets on queue */
  s8_t old_queue = ARP_TABLE_SIZE;
  /* its age */
  u8_t age_queue = 0;//记录old_queue的年龄

  /**
   * a) do a search through the cache, remember candidates
   * b) select candidate entry
   * c) create new entry
   */

  /* a) in a single search sweep, do all of this
   * 1) remember the first empty entry (if any)
   * 2) remember the oldest stable entry (if any)
   * 3) remember the oldest pending entry without queued packets (if any)
   * 4) remember the oldest pending entry with queued packets (if any)
   * 5) search for a matching IP entry, either pending or stable
   *    until 5 matches, or all entries are searched for.
   */
  //遍历所有的arp表
  for (i = 0; i < ARP_TABLE_SIZE; ++i) {
    u8_t state = arp_table[i].state;
    /* no empty entry found yet and now we do find one? */
    if ((empty == ARP_TABLE_SIZE) && (state == ETHARP_STATE_EMPTY)) {//记录索引号最小的empty表项
      /* remember first empty entry */
      empty = i;
    } else if (state != ETHARP_STATE_EMPTY) {//表被占用
      /* if given, does IP address match IP address in ARP entry? */
      if (ipaddr && ip_addr_cmp(ipaddr, &arp_table[i].ipaddr)) {//ip地址匹配直接返回
        /* found exact IP address match, simply bail out */
        return i;
      }
      /* pending entry? */
      if (state == ETHARP_STATE_PENDING) {//如果为pending
        /* pending with queued packets? */
        if (arp_table[i].q != NULL) {//pending且缓冲对列非空
          if (arp_table[i].ctime >= age_queue) {
            old_queue = i;
            age_queue = arp_table[i].ctime;//用old_queue和age_queue记录最老的那个非空pending索引号和年龄
          }
        } else //pending且缓冲对列为空
        /* pending without queued packets? */
        {
          if (arp_table[i].ctime >= age_pending) {
            old_pending = i;
            age_pending = arp_table[i].ctime;//用old_pending和age_pending记录最老的那个为空pending索引号和年龄
          }
        }
      /* stable entry? */
      } else if (state >= ETHARP_STATE_STABLE) {//如果为stable
        {
          /* remember entry with oldest stable entry in oldest, its age in maxtime */
          if (arp_table[i].ctime >= age_stable) {//用old_stable和age_stable记录最老的那个stable索引号和年龄
            old_stable = i;
            age_stable = arp_table[i].ctime;
          }
        }
      }
    }
  }
  /* { we have no match } => try to create a new entry *///到这里还没找到，则根据flag判断是否要创建表项
   
  /* don't create new entry, only search? */
  if (((flags & ETHARP_FLAG_FIND_ONLY) != 0) ||
      /* or no empty entry found and not allowed to recycle? */
      ((empty == ARP_TABLE_SIZE) && ((flags & ETHARP_FLAG_TRY_HARD) == 0))) {
    return (s8_t)ERR_MEM;
  }
  //需要创建表项，根据下面1)-->2)-->3)-->4)的重要程度来先回收后创建表项
  /* b) choose the least destructive entry to recycle:
   * 1) empty entry
   * 2) oldest stable entry
   * 3) oldest pending entry without queued packets
   * 4) oldest pending entry with queued packets
   * 
   * { ETHARP_FLAG_TRY_HARD is set at this point }
   */ 

  /* 1) empty entry available? */
  if (empty < ARP_TABLE_SIZE) {//首选empty
    i = empty;
  } else {
    /* 2) found recyclable stable entry? */
    if (old_stable < ARP_TABLE_SIZE) {//失败,则回收最老的stable表项
      /* recycle oldest stable*/
      i = old_stable;
    /* 3) found recyclable pending entry without queued packets? */
    } else if (old_pending < ARP_TABLE_SIZE) {//失败,则回收最老的pending且无数据包的表项
      /* recycle oldest pending */
      i = old_pending;
    /* 4) found recyclable pending entry with queued packets? */
    } else if (old_queue < ARP_TABLE_SIZE) {//失败,则回收最老的pending且有数据包的表项
      /* recycle oldest pending (queued packets are free in etharp_free_entry) */
      i = old_queue;
      /* no empty or recyclable entries found */
    } else {
      return (s8_t)ERR_MEM;
    }

    /* { empty or recyclable entry found } */
    etharp_free_entry(i);//释放老的表项所有数据包
  }

  /* IP address given? */
  if (ipaddr != NULL) {
    /* set IP address */
    ip_addr_copy(arp_table[i].ipaddr, *ipaddr);
  }
  arp_table[i].ctime = 0;
  return (err_t)i;//返回创建的表项
}

接着看etharp_update_arp_entry

/**
 * Update (or insert) a IP/MAC address pair in the ARP cache.
 *
 * If a pending entry is resolved, any queued packets will be sent
 * at this point.
 * 
 * @param netif netif related to this entry (used for NETIF_ADDRHINT)
 * @param ipaddr IP address of the inserted ARP entry.
 * @param ethaddr Ethernet address of the inserted ARP entry.
 * @param flags @see definition of ETHARP_FLAG_*
 *
 * @return
 * - ERR_OK Succesfully updated ARP cache.
 * - ERR_MEM If we could not add a new ARP entry when ETHARP_FLAG_TRY_HARD was set.
 * - ERR_ARG Non-unicast address given, those will not appear in ARP cache.
 *
 * @see pbuf_free()
 */
static err_t
etharp_update_arp_entry(struct netif *netif, ip_addr_t *ipaddr, struct eth_addr *ethaddr, u8_t flags)
{
  s8_t i;
  /* non-unicast address? */
  if (ip_addr_isany(ipaddr) ||//ip地址检查不能为0、广播、多播建立arp表
      ip_addr_isbroadcast(ipaddr, netif) ||
      ip_addr_ismulticast(ipaddr)) {
    return ERR_ARG;
  }
  /* find or create ARP entry */
  i = etharp_find_entry(ipaddr, flags);//查找或者回收新建arp表项
  /* bail out if no entry could be found */
  if (i < 0) {
    return (err_t)i;
  }

  {
    /* mark it stable */
    arp_table[i].state = ETHARP_STATE_STABLE;//记录为stable,因为下面就填mac地址了
  }

  /* record network interface */
  arp_table[i].netif = netif;

  /* update address */
  ETHADDR32_COPY(&arp_table[i].ethaddr, ethaddr);//填写mac地址
  /* reset time stamp */
  arp_table[i].ctime = 0;
  /* this is where we will send out queued packets! */

  while (arp_table[i].q != NULL) {//如果缓冲对列上有数据包则全部发送出去
    struct pbuf *p;
    /* remember remainder of queue */
    struct etharp_q_entry *q = arp_table[i].q;
    /* pop first item off the queue */
    arp_table[i].q = q->next;
    /* get the packet pointer */
    p = q->p;
    /* now queue entry can be freed */
    memp_free(MEMP_ARP_QUEUE, q);
    /* send the queued IP packet */
    etharp_send_ip(netif, p, (struct eth_addr*)(netif->hwaddr), ethaddr);
    /* free the queued IP packet */
    pbuf_free(p);
  }
  return ERR_OK;
}

ARP输出

当ip层要发送数据时，根据ip地址，对于广播、组播可以直接封装成帧发送，对于单播则需要查询arp表，找到则发送，找不到则要发送arp请求，并把数据包挂在缓冲对列上
ip_output会调用etharp_output，在etharp.c中

/**发送一个ip数据包pbuf到目的地址ipaddr，被ip层调用
 * Resolve and fill-in Ethernet address header for outgoing IP packet.
 *
 * For IP multicast and broadcast, corresponding Ethernet addresses
 * are selected and the packet is transmitted on the link.
 *
 * For unicast addresses, the packet is submitted to etharp_query(). In
 * case the IP address is outside the local network, the IP address of
 * the gateway is used.
 *
 * @param netif The lwIP network interface which the IP packet will be sent on.
 * @param q The pbuf(s) containing the IP packet to be sent.
 * @param ipaddr The IP address of the packet destination.
 *
 * @return
 * - ERR_RTE No route to destination (no gateway to external networks),
 * or the return type of either etharp_query() or etharp_send_ip().
 */
err_t
etharp_output(struct netif *netif, struct pbuf *q, ip_addr_t *ipaddr)
{
  struct eth_addr *dest;
  struct eth_addr mcastaddr;
  ip_addr_t *dst_addr = ipaddr;//目的ip

  LWIP_ASSERT("netif != NULL", netif != NULL);
  LWIP_ASSERT("q != NULL", q != NULL);
  LWIP_ASSERT("ipaddr != NULL", ipaddr != NULL);

  /* make room for Ethernet header - should not fail */
  if (pbuf_header(q, sizeof(struct eth_hdr)) != 0) {//调整payload使其指向以太网帧首部
    /* bail out */
    LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_LEVEL_SERIOUS,
      ("etharp_output: could not allocate room for header.\n"));
    LINK_STATS_INC(link.lenerr);
    return ERR_BUF;
  }

  /* Determine on destination hardware address. Broadcasts and multicasts
   * are special, other IP addresses are looked up in the ARP table. */

  /* broadcast destination IP address? */
  if (ip_addr_isbroadcast(ipaddr, netif)) {//目的ip是广播
    /* broadcast on Ethernet also */
    dest = (struct eth_addr *)&ethbroadcast;//全ff的mac
  /* multicast destination IP address? */
  } else if (ip_addr_ismulticast(ipaddr)) {//目的ip是多播
    /* Hash IP multicast address to MAC address.*/
    mcastaddr.addr[0] = LL_MULTICAST_ADDR_0;
    mcastaddr.addr[1] = LL_MULTICAST_ADDR_1;
    mcastaddr.addr[2] = LL_MULTICAST_ADDR_2;
    mcastaddr.addr[3] = ip4_addr2(ipaddr) & 0x7f;
    mcastaddr.addr[4] = ip4_addr3(ipaddr);
    mcastaddr.addr[5] = ip4_addr4(ipaddr);//构造多播mac
    /* destination Ethernet address is multicast */
    dest = &mcastaddr;
  /* unicast destination IP address? */
  } else {  //单播
    s8_t i;
    /* outside local network? if so, this can neither be a global broadcast nor
       a subnet broadcast. */
    if (!ip_addr_netcmp(ipaddr, &(netif->ip_addr), &(netif->netmask)) &&
        !ip_addr_islinklocal(ipaddr)) {//和网卡不在同一网段却ip不是169.254.0.0网段则要修改目的ip为默认网关
      {
        /* interface has default gateway? */
        if (!ip_addr_isany(&netif->gw)) {
          /* send to hardware address of default gateway IP address */
          dst_addr = &(netif->gw);
        /* no default gateway available */
        } else {
          /* no route to destination error (default gateway missing) */
          return ERR_RTE;
        }
      }
    }
    /* no stable entry found, use the (slower) query function:
       queue on destination Ethernet address belonging to ipaddr */
    return etharp_query(netif, dst_addr, q);//查询dst_addr对应的mac地址并发送数据包
  }

  /* continuation for multicast/broadcast destinations */
  /* obtain source Ethernet address of the given interface */
  /* send packet directly on the link */
  return etharp_send_ip(netif, q, (struct eth_addr*)(netif->hwaddr), dest);//直接发送
}

etharp_query

/**
 * Send an ARP request for the given IP address and/or queue a packet.
 *
 * If the IP address was not yet in the cache, a pending ARP cache entry
 * is added and an ARP request is sent for the given address. The packet
 * is queued on this entry.
 *
 * If the IP address was already pending in the cache, a new ARP request
 * is sent for the given address. The packet is queued on this entry.
 *
 * If the IP address was already stable in the cache, and a packet is
 * given, it is directly sent and no ARP request is sent out. 
 * 
 * If the IP address was already stable in the cache, and no packet is
 * given, an ARP request is sent out.
 * 
 * @param netif The lwIP network interface on which ipaddr
 * must be queried for.
 * @param ipaddr The IP address to be resolved.
 * @param q If non-NULL, a pbuf that must be delivered to the IP address.
 * q is not freed by this function.
 *
 * @note q must only be ONE packet, not a packet queue!
 *
 * @return
 * - ERR_BUF Could not make room for Ethernet header.
 * - ERR_MEM Hardware address unknown, and no more ARP entries available
 *   to query for address or queue the packet.
 * - ERR_MEM Could not queue packet due to memory shortage.
 * - ERR_RTE No route to destination (no gateway to external networks).
 * - ERR_ARG Non-unicast address given, those will not appear in ARP cache.
 *
 */
err_t
etharp_query(struct netif *netif, ip_addr_t *ipaddr, struct pbuf *q)
{
  struct eth_addr * srcaddr = (struct eth_addr *)netif->hwaddr;
  err_t result = ERR_MEM;
  s8_t i; /* ARP entry index */

  /* non-unicast address? */
  if (ip_addr_isbroadcast(ipaddr, netif) ||
      ip_addr_ismulticast(ipaddr) ||
      ip_addr_isany(ipaddr)) {
    LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: will not add non-unicast IP address to ARP cache\n"));
    return ERR_ARG;
  }

  /* find entry in ARP cache, ask to create entry if queueing packet */
  i = etharp_find_entry(ipaddr, ETHARP_FLAG_TRY_HARD);//查找或者回收创建表项

  /* could not find or create entry? */
  if (i < 0) {
    LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not create ARP entry\n"));
    if (q) {
      LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: packet dropped\n"));
      ETHARP_STATS_INC(etharp.memerr);
    }
    return (err_t)i;
  }

  /* mark a fresh entry as pending (we just sent a request) */
  if (arp_table[i].state == ETHARP_STATE_EMPTY) {//如果是empty，则说明是刚创建的，变成pending
    arp_table[i].state = ETHARP_STATE_PENDING;
  }

  /* { i is either a STABLE or (new or existing) PENDING entry } */
  LWIP_ASSERT("arp_table[i].state == PENDING or STABLE",
  ((arp_table[i].state == ETHARP_STATE_PENDING) ||
   (arp_table[i].state >= ETHARP_STATE_STABLE)));

  /* do we have a pending entry? or an implicit query request? */
  if ((arp_table[i].state == ETHARP_STATE_PENDING) || (q == NULL)) {
    /* try to resolve it; send out ARP request */
    result = etharp_request(netif, ipaddr);//pending或者数据包为空则发送arp请求
    if (result != ERR_OK) {
      /* ARP request couldn't be sent */
      /* We don't re-send arp request in etharp_tmr, but we still queue packets,
         since this failure could be temporary, and the next packet calling
         etharp_query again could lead to sending the queued packets. */
    }
    if (q == NULL) {
      return result;
    }
  }
  //数据包不为空
  /* packet given? */
  LWIP_ASSERT("q != NULL", q != NULL);
  /* stable entry? */
  if (arp_table[i].state >= ETHARP_STATE_STABLE) {//arp stable,则直接发送数据包
    /* we have a valid IP->Ethernet address mapping */
    ETHARP_SET_HINT(netif, i);
    /* send the packet */
    result = etharp_send_ip(netif, q, srcaddr, &(arp_table[i].ethaddr));
  /* pending entry? (either just created or already pending */
  } else if (arp_table[i].state == ETHARP_STATE_PENDING) {//arp pending 挂载数据包
    /* entry is still pending, queue the given packet 'q' */
    struct pbuf *p;
    int copy_needed = 0;
    /* IF q includes a PBUF_REF, PBUF_POOL or PBUF_RAM, we have no choice but
     * to copy the whole queue into a new PBUF_RAM (see bug #11400) 
     * PBUF_ROMs can be left as they are, since ROM must not get changed. */
    p = q;//对于挂载的数据报，非PBUF_ROMs类型都需要copy
    while (p) {
      LWIP_ASSERT("no packet queues allowed!", (p->len != p->tot_len) || (p->next == 0));
      if(p->type != PBUF_ROM) {
        copy_needed = 1;
        break;
      }
      p = p->next;
    }
    if(copy_needed) {
      /* copy the whole packet into new pbufs */
      p = pbuf_alloc(PBUF_RAW, p->tot_len, PBUF_RAM);
      if(p != NULL) {
        if (pbuf_copy(p, q) != ERR_OK) {
          pbuf_free(p);
          p = NULL;
        }
      }
    } else {
      /* referencing the old pbuf is enough */
      p = q;
      pbuf_ref(p);
    }
    /* packet could be taken over? */
    if (p != NULL) {
      /* queue packet ... */
#if ARP_QUEUEING
      struct etharp_q_entry *new_entry;
      /* allocate a new arp queue entry */
      new_entry = (struct etharp_q_entry *)memp_malloc(MEMP_ARP_QUEUE);
      if (new_entry != NULL) {
        new_entry->next = 0;
        new_entry->p = p;
        if(arp_table[i].q != NULL) {
          /* queue was already existent, append the new entry to the end */
          struct etharp_q_entry *r;
          r = arp_table[i].q;
          while (r->next != NULL) {
            r = r->next;
          }
          r->next = new_entry;
        } else {
          /* queue did not exist, first item in queue */
          arp_table[i].q = new_entry;
        }
        LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet %p on ARP entry %"S16_F"\n", (void *)q, (s16_t)i));
        result = ERR_OK;
      } else {
        /* the pool MEMP_ARP_QUEUE is empty */
        pbuf_free(p);
        LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet %p (out of memory)\n", (void *)q));
        result = ERR_MEM;
      }
#else /* ARP_QUEUEING */
      /* always queue one packet per ARP request only, freeing a previously queued packet */
      if (arp_table[i].q != NULL) {
        LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: dropped previously queued packet %p for ARP entry %"S16_F"\n", (void *)q, (s16_t)i));
        pbuf_free(arp_table[i].q);
      }
      arp_table[i].q = p;
      result = ERR_OK;
      LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: queued packet %p on ARP entry %"S16_F"\n", (void *)q, (s16_t)i));
#endif /* ARP_QUEUEING */
    } else {
      ETHARP_STATS_INC(etharp.memerr);
      LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_query: could not queue a copy of PBUF_REF packet %p (out of memory)\n", (void *)q));
      result = ERR_MEM;
    }
  }
  return result;
}

etharp_send_ip

/**
 * Send an IP packet on the network using netif->linkoutput
 * The ethernet header is filled in before sending.
 *
 * @params netif the lwIP network interface on which to send the packet
 * @params p the packet to send, p->payload pointing to the (uninitialized) ethernet header
 * @params src the source MAC address to be copied into the ethernet header
 * @params dst the destination MAC address to be copied into the ethernet header
 * @return ERR_OK if the packet was sent, any other err_t on failure
 */
static err_t
etharp_send_ip(struct netif *netif, struct pbuf *p, struct eth_addr *src, struct eth_addr *dst)
{
  struct eth_hdr *ethhdr = (struct eth_hdr *)p->payload;

  LWIP_ASSERT("netif->hwaddr_len must be the same as ETHARP_HWADDR_LEN for etharp!",
              (netif->hwaddr_len == ETHARP_HWADDR_LEN));
  ETHADDR32_COPY(&ethhdr->dest, dst);
  ETHADDR16_COPY(&ethhdr->src, src);
  ethhdr->type = PP_HTONS(ETHTYPE_IP);
  LWIP_DEBUGF(ETHARP_DEBUG | LWIP_DBG_TRACE, ("etharp_send_ip: sending packet %p\n", (void *)p));
  /* send the packet */
  return netif->linkoutput(netif, p);//调用网卡直接发送
}

ARP层总流程如下图