作者:lwyang?
内核版本:Linux-4.20.8
网络子系统中用来存储数据的缓冲区叫做套接字缓存,简称SKB
,可处理变长数据,尽量避免数据的复制。
每一个SKB都在设备中标识发送报文的目的或接受报文的来源地,主要用于在网络驱动程序和应用程序直接传递复制数据包。
当应用程序要发送一个数据包,数据通过系统调用提交到内核,系统分配一个SKB来存储数据,然后往下层传递,在传递到网络驱动后才将其释放。当网路设备接受到数据包,同样分配一个SKB来存储数据,然后向上传递,最终在数据复制到应用程序后释放。
SKB 有两部分,一部分为SKB描述符(sk_buff结构本身),另一部分为数据缓冲区(sk_buff的head指向)
struct sk_buff {
union {
struct {
/* These two members must be first. */
//这里为什么Next,Previous要在结构体的第一个,后面会解释
//Next buffer in list
struct sk_buff *next;
//Previous buffer in list
struct sk_buff *prev;
union {
//表示与SKB相关联的网络接口设备,也称为网络接口卡(NIC)
struct net_device *dev;
/* Some protocols might use this space to store information,
* while device pointer would be NULL.
* UDP receive path is one user.
*/
unsigned long dev_scratch;
};
};
//红黑树节点,RB tree node, alternative to next/prev for netem/tcp
struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */
struct list_head list;
};
union {
//Socket we are owned by,对于本地生成的流量或发送给当前主机的流量,sk为拥有skb的套接字,对于需要转发的数据包,sk为NULL
//skb_orphan(struct sk_buff *skb),如果指定skb有destructor,就调用它,将指定sock对象(sk)为NULL,并将destructor设置为NULL
struct sock *sk;
int ip_defrag_offset;
};
union {
//数据包到达的时间,在skb中,存储的时间戳为相对于参考时间的偏移量。不要将tstamp与硬件时间混为一谈,后者是使用skb_shared_info的成员hwtstamps实现的
ktime_t tstamp;
u64 skb_mstamp_ns; /* earliest departure time */
};
/*
* This is the control buffer. It is free to use for every
* layer. Please put your private variables there. If you
* want to keep them across layers you have to do a skb_clone()
* first. This is owned by whoever has the skb queued ATM.
*/
//控制缓冲区,可供任何层使用,不透明区域,用于存储专用信息
char cb[48] __aligned(8);
union {
struct {
//destination entry (with norefcount bit)
//目的条目地址(dst_entry),表示目的地的路由选择条目,对于每个数据包都需要执行路由选择表查找,查找结构决定了如何处理数据包
//skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) 设置skb的dst
//可能会对指向的对象dst进行了引用计数,如果没有进行引用计数,_skb_refdst最后一位将为1
unsigned long _skb_refdst;
//
void (*destructor)(struct sk_buff *skb);
};
//list structure for TCP (tp->tsorted_sent_queue)
struct list_head tcp_tsorted_anchor;
};
#ifdef CONFIG_XFRM
//安全路径指针,包含IPsec XFRM变换状态(xfrm_state)数组,IPsec是一种3层协议,主要用于VPN,IPv6中必须实现
//struct sec_path *skb_sec_path(struct sk_buff *skb) 返回相关联的sec_path对象
struct sec_path *sp;
#endif
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
//Associated connection, if any (with nfctinfo bits),连接跟踪信息,让内核能够跟踪所有网络连接和会话
unsigned long _nfct;
#endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info *nf_bridge;
#endif
//len:数据包总字节数
//data_len:非线性数据长度,有分页数据,paged data时才使用这个字段
//bool skb_is_nonlinear(const struct sk_buff *skb) 在指定skb的data_len大于0时返回true
unsigned int len,
data_len;
//mac_len:MAC(2层)包头长度
//hdr_len:writable header length of cloned skb
__u16 mac_len,
hdr_len;
/* Following fields are _not_ copied in __copy_skb_header()
* Note that queue_mapping is here mostly to fill a hole.
*/
//Queue mapping for multiqueue devices
__u16 queue_mapping;
/* if you move cloned around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define CLONED_MASK (1 << 7)
#else
#define CLONED_MASK 1
#endif
#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset)
__u8 __cloned_offset[0];
//使用__skb_clone()克隆数据包时,被克隆和克隆得到的数据包中,这个字段都被置为1
__u8 cloned:1,
//Payload reference only, must not modify header,只考虑有效载荷,禁止修改包头
nohdr:1,
//skbuff clone status
//SKB_FCLONE_UNAVAILABLE skb未被克隆
//SKB_FCLONE_ORIG 在skbuff_fclone_cache分配的附skb,可以被克隆
//SKB_FCLONE_CLONE 在skbuff_fclone_cache分配的子skb,从父skb克隆得到的
fclone:2,
//this packet has been seen already, so stats have been done for it, don’t do them again
peeked:1,
head_frag:1,
//More SKBs are pending for this queue
xmit_more:1,
//skbuff was allocated from PFMEMALLOC reserves
pfmemalloc:1;
/* fields enclosed in headers_start/headers_end are copied
* using a single memcpy() in __copy_skb_header()
*/
/* private: */
__u32 headers_start[0];
/* public: */
/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX (7 << 5)
#else
#define PKT_TYPE_MAX 7
#endif
#define PKT_TYPE_OFFSET() offsetof(struct sk_buff, __pkt_type_offset)
__u8 __pkt_type_offset[0];
//对于以太网,数据包类型取决于以太网包头的目的mac地址,并由eth_type_trans()确定
//PACKET_BROADCAST 广播
//PACKET_MULTICAST 组播
//PACKET_HOST 目的MAC为作为参数传入设备的MAC地址
//PACKET_OTHERHOST 上述条件都不满足
__u8 pkt_type:3;
//allow local fragmentation
__u8 ignore_df:1;
//netfilter packet trace flag
__u8 nf_trace:1;
//Driver fed us an IP checksum
__u8 ip_summed:2;
//allow the mapping of a socket to a queue to be changed
__u8 ooo_okay:1;
//indicate hash is a canonical 4-tuple hash over transport ports.
__u8 l4_hash:1;
//indicates hash was computed in software stack
__u8 sw_hash:1;
//wifi_acked was set
__u8 wifi_acked_valid:1;
//whether frame was acked on wifi or not
__u8 wifi_acked:1;
//Request NIC to treat last 4 bytes as Ethernet FCS
__u8 no_fcs:1;
/* Indicates the inner headers are valid in the skbuff. */
//指出SKB是用于封装的,例如,VXLAN驱动程序就是用这个字段,VXLAN是一种通过CPU内核套接字传输2层以太网数据包的协议,可在防火墙阻断了隧道,只让TCP或UDP流量通过时提供解决方案
__u8 encapsulation:1;
__u8 encap_hdr_csum:1;
__u8 csum_valid:1;
__u8 csum_complete_sw:1;
__u8 csum_level:2;
//use CRC32c to resolve CHECKSUM_PARTIAL
__u8 csum_not_inet:1;
//need to confirm neighbour
__u8 dst_pending_confirm:1;
#ifdef CONFIG_IPV6_NDISC_NODETYPE
//router type (from link layer)
__u8 ndisc_nodetype:2;
#endif
//skb是否归ipvs(IP虚拟服务器)所有,ipvs是一种基于内核传输层负载均衡解决方案
__u8 ipvs_property:1;
__u8 inner_protocol_type:1;
__u8 remcsum_offload:1;
#ifdef CONFIG_NET_SWITCHDEV
//Packet was L2-forwarded in hardware
__u8 offload_fwd_mark:1;
__u8 offload_mr_fwd_mark:1;
#endif
#ifdef CONFIG_NET_CLS_ACT
//do not classify packet. set by IFB device
__u8 tc_skip_classify:1;
//used within tc_classify to distinguish in/egress
__u8 tc_at_ingress:1;
//packet was redirected by a tc action
__u8 tc_redirected:1;
//if tc_redirected, tc_at_ingress at time of redirect
__u8 tc_from_ingress:1;
#endif
#ifdef CONFIG_TLS_DEVICE
//Decrypted SKB
__u8 decrypted:1;
#endif
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
#endif
union {
//校验和Checksum (must include start/offset pair)
__wsum csum;
struct {
//Offset from skb->head where checksumming should start
__u16 csum_start;
//Offset from csum_start where checksum should be stored
__u16 csum_offset;
};
};
//数据包的排队优先级,在接受路径中,skb的优先级是根据套接字的优先级(套接字的sk_priority)设置的。对于转发的数据包,优先级是根据ip包头的TOS设置的
__u32 priority;
//数据包到达的网络设备的ifindex
int skb_iif;
//the packet hash
__u32 hash;
//使用的vlan协议,通常为802.1q
__be16 vlan_proto;
//vlan标记控制信息(2字节),有ID和优先级组成
__u16 vlan_tci;
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
union {
//id of the NAPI struct this skb came from
unsigned int napi_id;
unsigned int sender_cpu;
};
#endif
#ifdef CONFIG_NETWORK_SECMARK
//security marking,安全标记字段,由iptables SECMARK目标设置
__u32 secmark;
#endif
union {
//通过标识来标记SKB,在iptables中使用MARK目标和managle表来设置mark字段
//iptables -A PREROUTING -t manage -i eth1 -j MARK --set-mark 0x1234
__u32 mark;
//用于方法sk_stream_alloc_skb()中
__u32 reserved_tailroom;
};
union {
//Protocol (encapsulation)
__be16 inner_protocol;
__u8 inner_ipproto;
};
//Inner transport layer header (encapsulation)
__u16 inner_transport_header;
//Network layer header (encapsulation)
__u16 inner_network_header;
//Link layer header (encapsulation)
__u16 inner_mac_header;
//协议字段,在使用以太网和IP时,在接收路径中由方法eth_type_trans()设置为ETH_P_IP
__be16 protocol;
//传输层(L4)报头
__u16 transport_header;
//网络层(L3)报头
__u16 network_header;
//数据链路(层L2)报头,比如要获取二层头部:skb->head + skb->mac_header
__u16 mac_header;
/* private: */
__u32 headers_end[0];
/* public: */
/* These elements must be at the end, see alloc_skb() for details. */
//数据尾
sk_buff_data_t tail;
//缓冲区末尾
sk_buff_data_t end;
//head:缓冲区开头
//data:数据头
unsigned char *head,
*data;
//为SKB分配的总内存(包括SKB结构本身以及分配的数据块长度)
unsigned int truesize;
//引用计数器,初始化为1
//skb_get(struct sk_buff *skb) 将引用计数器加1
//skb_shared(const struct sk_buff *skb) 如果users不为1,就返回true
//skb_share_check(struct sk_buff *skb, gfp_t pri) 如果缓冲区未被共享,就返回原始缓冲区,如果缓冲区被共享,就复制它,将原始缓冲区引用计数减1,并返回新复制的缓冲区。在中断上下文中调度或持有自旋锁时,参数pri必须为GFP_ATOMIC
refcount_t users;
};
FAQ:为什么Next,Previous指针要放在结构体的开头?
先看sk_buff_head
这个结构体
struct sk_buff_head {
/* These two members must be first. */
struct sk_buff *next;
struct sk_buff *prev;
//sk_buff 链表长度
__u32 qlen;
//防止对sk_buff链表的并发访问
spinlock_t lock;
};
对链表头的初始化
static inline void skb_queue_head_init(struct sk_buff_head *list)
{
//自旋锁的初始化
spin_lock_init(&list->lock);
__skb_queue_head_init(list);
}
static inline void __skb_queue_head_init(struct sk_buff_head *list)
{
//先将sk_buff_head 强转为sk_buff 结构,让list->prev,list->next指向sk_buff
//因为这两个结构体前两个元素相同,因此可以将sk_buff_head 强转为sk_buff 获取next,prev节点
//因为后续链表操作只会操作prev,next这两个元素,这个强转不会对结构体中其他元素造成影响
//因此,其实prev,next并不一定要在结构体开头,只要sk_buff_head 和sk_buff 中prev,next相对于结构体开头的偏移量相同就行
list->prev = list->next = (struct sk_buff *)list;
list->qlen = 0;
}
因此,其实prev,next
并不一定要在结构体开头,只要sk_buff_head
和sk_buff
中prev,next
相对于结构体开头的偏移量相同就行,再看插入sk_buff操作就会明白了
在链表头插入新节点sk_buff
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
{
unsigned long flags;
spin_lock_irqsave(&list->lock, flags);
__skb_queue_head(list, newsk);
spin_unlock_irqrestore(&list->lock, flags);
}
static inline void __skb_queue_head(struct sk_buff_head *list,
struct sk_buff *newsk)
{
//这里也是将sk_buff_head 强转为sk_buff 结构,方便后续调用__skb_insert
__skb_queue_after(list, (struct sk_buff *)list, newsk);
}
static inline void __skb_queue_after(struct sk_buff_head *list,
struct sk_buff *prev,
struct sk_buff *newsk)
{
__skb_insert(newsk, prev, prev->next, list);
}
static inline void __skb_insert(struct sk_buff *newsk,
struct sk_buff *prev, struct sk_buff *next,
struct sk_buff_head *list)
{
//在链表头插入新节点newsk
newsk->next = next;
newsk->prev = prev;
next->prev = prev->next = newsk;
//将链表节点数量加1
list->qlen++;
}
FAQ 【完】
在数据缓冲区的末尾(skb_end_pointer(SKB)
),即end指针指向的地址紧跟着一个skb_shared_info
结构,保存着数据块的附加信息
/* This data is invariant across clones and lives at
* the end of the header data, ie. at skb->end.
*/
struct skb_shared_info {
__u8 __unused;
__u8 meta_len;
//数组frags包含的元素个数
__u8 nr_frags;
/* generate hardware time stamp */
//SKBTX_HW_TSTAMP = 1 << 0,
/* generate software time stamp when queueing packet to NIC */
//SKBTX_SW_TSTAMP = 1 << 1,
/* device driver is going to provide hardware time stamp */
//SKBTX_IN_PROGRESS = 1 << 2,
/* device driver supports TX zero-copy buffers */
//SKBTX_DEV_ZEROCOPY = 1 << 3,
/* generate wifi status information (where possible) */
//SKBTX_WIFI_STATUS = 1 << 4,
/* This indicates at least one fragment might be overwritten
* (as in vmsplice(), sendfile() ...)
* If we need to compute a TX checksum, we'll need to copy
* all frags to avoid possible bad checksum
*/
//SKBTX_SHARED_FRAG = 1 << 5,
/* generate software time stamp when entering packet scheduling */
//SKBTX_SCHED_TSTAMP = 1 << 6,
__u8 tx_flags;
unsigned short gso_size;
/* Warning: this field is not always filled in (UFO)! */
unsigned short gso_segs;
struct sk_buff *frag_list;
struct skb_shared_hwtstamps hwtstamps;
unsigned int gso_type;
u32 tskey;
/*
* Warning : all fields before dataref are cleared in __alloc_skb()
*/
//结构skb_shared_info 的引用计数器
atomic_t dataref;
/* Intermediate layers must ensure that destructor_arg
* remains valid until skb destructor */
void * destructor_arg;
/* must be last field, see pskb_expand_head() */
skb_frag_t frags[MAX_SKB_FRAGS];
};
typedef struct skb_frag_struct skb_frag_t;
struct skb_frag_struct {
struct {
//指向文件系统缓存页的指针
struct page *p;
} page;
#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
//数据起始地址在文件系统缓存页中的偏移
__u32 page_offset;
//数据在文件系统缓存页中使用的长度
__u32 size;
#else
__u16 page_offset;
__u16 size;
#endif
};
nr_frags,frags,frag_list
与IP分片存储有关。通常数据存储在线性区域中,但当为了支持聚合分散I/O,frags,frag_list
支持聚合分散I/O。
frag_list
的用法:
static inline bool skb_is_nonlinear(const struct sk_buff *skb)
{
return skb->data_len;
}
skb_is_nonlinear
就是用来判断SKB是否存在非线性缓冲区,实际上就是判断data_len
成员
没有启用分片的报文,数据长度len
为x,即data到tail的长度,nr_frags
为0,frag_list
为NULL
启用聚合分散I/O的报文,数据长度len
为x+S1+S2,x为data到tail的长度,S1和S2分别为两个分片的长度,data_len
为S1+S2,表示存在聚合分散I/O数据。nr_frags
为2,而frag_list
为NULL,说明这不是普通的分片,而是聚合分散I/O分片,数量为2,这两个分片指向同一物理分页,各自在分页中的偏移和长度分别是0/S1和S1/S2
使用FRAGLIST类型的分散聚合I/O报文,数据长度len
为x+S1,而S1为FRAGLIST类型分散聚合I/O数据长度,data_len
为S1,表示存在分散聚合I/O数据。nr_frags
为0,而frag_list
不为NULL,这表明存在FRAGLIST类型分散聚合I/O数据
在sk_buff中没有指向skb_shared_info
结构的指针,可是用skb_shinfo
宏来访问
/* Internal */
#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
普通分散/聚合IO(nr_frags和frags组成),只是让程序和硬件可以使用非相邻内存区域,就好像它们是相邻的那样,frags里的数据是主缓冲区中(head-end)数据的扩展(在ip_append_data中更新)。
FRAGLIST类型的分散/聚合IO(frag_list)里的数据代表的是独立缓冲区,也就是每一个缓冲区都必须作为单独的IP片段进行独立传输(在ip_push_pending_frames中更新)
?对SKB的操作请看下节《Linux内核网络子系统源码解析(二)----------sk_buff的操作》?