IPVS负载均衡（六）浅析ip_vs_core.c
IPVS负载均衡（六）浅析ip_vs_core.c
ip_vs_core.c
先从linux内核module_init和module_exit开始。
module_init
函数：ip_vs_init
函数原型：static int __init ip_vs_init(void)
初始化ip_vs模块所需的各种
/*
* Initialize IP Virtual Server
*/
static int __init ip_vs_init(void)
{
int ret;
ret = ip_vs_control_init();
if (ret < 0) {
pr_err("can't setup control.\n");
goto exit;
}
ip_vs_protocol_init();
ret = ip_vs_conn_init();
if (ret < 0) {
pr_err("can't setup connection table.\n");
goto cleanup_protocol;
}
ret = register_pernet_subsys(&ipvs_core_ops); /* Alloc ip_vs struct */
if (ret < 0)
goto cleanup_conn;
ret = register_pernet_device(&ipvs_core_dev_ops);
if (ret < 0)
goto cleanup_sub;
ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
if (ret < 0) {
pr_err("can't register hooks.\n");
goto cleanup_dev;
}
ret = ip_vs_register_nl_ioctl();
if (ret < 0) {
pr_err("can't register netlink/ioctl.\n");
goto cleanup_hooks;
}
pr_info("ipvs loaded.\n");
return ret;
cleanup_hooks:
nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
cleanup_dev:
unregister_pernet_device(&ipvs_core_dev_ops);
cleanup_sub:
unregister_pernet_subsys(&ipvs_core_ops);
cleanup_conn:
ip_vs_conn_cleanup();
cleanup_protocol:
ip_vs_protocol_cleanup();
ip_vs_control_cleanup();
exit:
return ret;
}主要调用了：
->ip_vs_control_init
->ip_vs_protocol_init，协议所需初始化
->ip_vs_conn_init，链接管理初始化
->register_pernet_subsys
->register_pernet_device
->nf_register_hooks，注册netfilter钩子函数，后续代码流程会从这里开始
->ip_vs_register_nl_ioctl其中跟IPVS主要流程最相关的是nf_register_hooks、ip_vs_protocol_init、ip_vs_conn_init，后续会主要从nf_register_hooks这里–在netfilter上注册的钩子函数上梳理主干流程
module_exit
函数：ip_vs_cleanup
函数原型：static void __exit ip_vs_cleanup(void)
卸载ip_vs模块，释放占用的资源等操作
static void __exit ip_vs_cleanup(void)
{
ip_vs_unregister_nl_ioctl();
nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
unregister_pernet_device(&ipvs_core_dev_ops);
unregister_pernet_subsys(&ipvs_core_ops); /* free ip_vs struct */
ip_vs_conn_cleanup();
ip_vs_protocol_cleanup();
ip_vs_control_cleanup();
pr_info("ipvs unloaded.\n");
}主要调用：
->ip_vs_unregister_nl_ioctl
->nf_unregister_hooks
->unregister_pernet_device
->unregister_pernet_subsys
->ip_vs_conn_cleanup
->ip_vs_protocol_cleanup
->ip_vs_control_cleanup
可以注意到，ip_vs_cleanup资源释放顺序，与ip_vs_init申请资源顺序刚好相反；且与ip_vs_init中的出错处理部分顺序基本相同。
看完模块注册去注册流程，那么后面则开始寻找
nf_register_hooks钩子函数
通过nf_register_hooks注册的钩子函数进一步梳理代码流程
ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
ip_vs_ops的定义
static struct nf_hook_ops ip_vs_ops[] __read_mostly = {
/* After packet filtering, change source only for VS/NAT */
{
.hook = ip_vs_reply4,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_LOCAL_IN,
.priority = NF_IP_PRI_NAT_SRC - 2,
},
/* After packet filtering, forward packet through VS/DR, VS/TUN,
* or VS/NAT(change destination), so that filtering rules can be
* applied to IPVS. */
{
.hook = ip_vs_remote_request4,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_LOCAL_IN,
.priority = NF_IP_PRI_NAT_SRC - 1,
},
/* Before ip_vs_in, change source only for VS/NAT */
{
.hook = ip_vs_local_reply4,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_LOCAL_OUT,
.priority = NF_IP_PRI_NAT_DST + 1,
},
/* After mangle, schedule and forward local requests */
{
.hook = ip_vs_local_request4,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_LOCAL_OUT,
.priority = NF_IP_PRI_NAT_DST + 2,
},
/* After packet filtering (but before ip_vs_out_icmp), catch icmp
* destined for 0.0.0.0/0, which is for incoming IPVS connections */
{
.hook = ip_vs_forward_icmp,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_FORWARD,
.priority = 99,
},
/* After packet filtering, change source only for VS/NAT */
{
.hook = ip_vs_reply4,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_FORWARD,
.priority = 100,
},
#ifdef CONFIG_IP_VS_IPV6
/* After packet filtering, change source only for VS/NAT */
{
.hook = ip_vs_reply6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_LOCAL_IN,
.priority = NF_IP6_PRI_NAT_SRC - 2,
},
/* After packet filtering, forward packet through VS/DR, VS/TUN,
* or VS/NAT(change destination), so that filtering rules can be
* applied to IPVS. */
{
.hook = ip_vs_remote_request6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_LOCAL_IN,
.priority = NF_IP6_PRI_NAT_SRC - 1,
},
/* Before ip_vs_in, change source only for VS/NAT */
{
.hook = ip_vs_local_reply6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_LOCAL_OUT,
.priority = NF_IP6_PRI_NAT_DST + 1,
},
/* After mangle, schedule and forward local requests */
{
.hook = ip_vs_local_request6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_LOCAL_OUT,
.priority = NF_IP6_PRI_NAT_DST + 2,
},
/* After packet filtering (but before ip_vs_out_icmp), catch icmp
* destined for 0.0.0.0/0, which is for incoming IPVS connections */
{
.hook = ip_vs_forward_icmp_v6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_FORWARD,
.priority = 99,
},
/* After packet filtering, change source only for VS/NAT */
{
.hook = ip_vs_reply6,
.pf = NFPROTO_IPV6,
.hooknum = NF_INET_FORWARD,
.priority = 100,
},
#endif
};由于是浅析，故这里就不会分析IPV6的场景，主要看ipv4的场景。
.hook涉及到的函数有：
- ip_vs_reply4
- ip_vs_remote_request4
- ip_vs_local_reply4
- ip_vs_local_request4
- ip_vs_forward_icmp
- ip_vs_reply4
netfilter钩子函数最后返回值主要有这几种：
| 名称 | 含义 |
|---|---|
| NF_DROP | 丢弃该数据包 |
| NF_ACCEPT | 保留该数据包 |
| NF_STOLEN | 忘掉该数据包 |
| NF_QUEUE | 将该数据包插入到用户空间 |
| NF_REPEAT | 再次调用该hook函数 |
.pf是IP类型，这里只讨论IPV4的情况
.hooknum则是涉及到netfilter的五处钩子点，简单可以理解为，netfilter在内核协议栈中五处地方有函数指针，如果注册了钩子函数，则会在对应点调用指定的钩子函数，处理报文。netfilter的五处分别为：NF_IP_PRE_ROUTING，在完整性校验之后，选路确定之前；NF_IP_LOCAL_IN，在选路确定之后，且数据包的目的是本地主机；NF_IP_FORWARD，目的地是其它主机地数据包；NF_IP_LOCAL_OUT，来自本机进程的数据包在其离开本地主机的过程中；NF_IP_POST_ROUTING，在数据包离开本地主机“上线”之前。
可以看到在ipvs中，主要用到了NF_IP_LOCAL_IN、NF_IP_LOCAL_OUT、NF_IP_FORWARD。
.priority则是表示优先级，因为可能会挂多个钩子函数，所以设置优先级，会先调用优先级高的，再调用优先级低的进行处理。
下一步
可以看到ip_vs_reply4、ip_vs_local_reply4都是调用了ip_vs_out
ip_vs_remote_request4、ip_vs_local_request4都是调用了ip_vs_in
下面将会简要分析ip_vs_out和ip_vs_in两个函数的处理流程