云原生边缘计算解决方案之openyurt
架构解析(openyurt)
组件
- master组件：
- yurt-controller-manager
- yurt-tunnel-server
- yurt app manager
- 边缘worker组件
- yurt-tunnel-agent
- yurt-hub
tunnel原理解析
原生k8s：client(kubectl exec,log等) => kube-apiserver:6443 => kubelet-node:10250
边缘k8s： client(kubectl exec,log等) => kube-apiserver:6443 => yurt-tunnel-server:10263=> yurt-tunnel-agent => kubelet-node:10250
openyurt解决思路：通过iptables截取流量的方式解决 如下是iptables的链路转发的整个流程：
Chain OUTPUT:
TUNNEL-PORT tcp -- 0.0.0.0/0 0.0.0.0/0 /* yurttunnel server port */
Chain TUNNEL-PORT:
TUNNEL-PORT-10255 tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:10255 /* jump to port 10255 */
TUNNEL-PORT-10250 tcp -- 0.0.0.0/0 0.0.0.0/0 tcp dpt:10250 /* jump to port 10250 */
Chain TUNNEL-PORT-10255
RETURN tcp -- 0.0.0.0/0 127.0.0.1 /* return request to access node directly */ tcp dpt:10250
RETURN tcp -- 0.0.0.0/0 172.16.1.199 /* return request to access node directly */ tcp dpt:10250
DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263
Chain TUNNEL-PORT-10250
RETURN tcp -- 0.0.0.0/0 127.0.0.1 /* return request to access node directly */ tcp dpt:10250
RETURN tcp -- 0.0.0.0/0 172.16.1.199 /* return request to access node directly */ tcp dpt:10250
DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263
yurt-hub原理解析
作用：用作来自Kubernetes节点守护程序（Kubelet，Kubeproxy，CNI插件等）的出站流量的代理，并配有本地缓存机制，当节点无法通信apiserver的时候 不影响该节点正常提供业务达到边缘自治的功能并且yurt-hub守护程序可以帮助节点在重新启动后恢复状态。
目前版本缺陷 无法代理hostnetwork=true的pod
原因：为了保证安全性 yurthub只透出了127.0.0.1:10261端口。
解决：
- 使用hostnetwork=true模式
- 在无法使用hostnetwork=true模式的情况下 需要修改源码加入dummy类型网卡
代码实现：
首先创建TAP网卡，之后使用setupIfce修改网卡ip地址
func Create() (*taptun.Tap, string){
tap, err := taptun.NewTap(TapDeivceName)
if err != nil {
klog.Fatalf("%v:%v",TapDeivceName,err)
}
self := net.IPv4(169, 254, 2, 1)
mask := net.IPv4Mask(255, 255, 255, 255)
setupIfce(net.IPNet{IP: self, Mask: mask}, TapDeivceName)
return tap,self.String()
}
func setupIfce(ipNet net.IPNet, dev string) {
if err := exec.Command("ip", "link", "set", dev, "up").Run(); err != nil {
klog.Fatal(err)
}
if err := exec.Command("ip", "addr", "add", ipNet.String(), "dev", dev).Run(); err != nil {
klog.Fatal(err)
}
}
//创建网卡
ifce, ipaddr := tap.Create()
tapServer := &http.Server{
Addr: fmt.Sprintf("%v:10261", ipaddr),
Handler: proxyHandler,
MaxHeaderBytes: 1 << 20,
}
tapProxyServer := tapProxyServer{
ifce: ifce,
tapHttp: tapServer,
}将虚拟网卡绑定hub的proxy
func NewYurtHubServer(cfg *config.YurtHubConfiguration,
certificateMgr interfaces.YurtCertificateManager,
proxyHandler http.Handler) Server {
hubMux := mux.NewRouter()
registerHandlers(hubMux, cfg, certificateMgr)
hubServer := &http.Server{
Addr: cfg.YurtHubServerAddr,
Handler: hubMux,
MaxHeaderBytes: 1 << 20,
}
proxyServer := &http.Server{
Addr: cfg.YurtHubProxyServerAddr,
Handler: proxyHandler,
MaxHeaderBytes: 1 << 20,
}
//创建网卡
ifce, ipaddr := tap.Create()
tapServer := &http.Server{
Addr: fmt.Sprintf("%v:10261", ipaddr),
Handler: proxyHandler,
MaxHeaderBytes: 1 << 20,
}
tapProxyServer := tapProxyServer{
ifce: ifce,
tapHttp: tapServer,
}
return &yurtHubServer{
hubServer: hubServer,
proxyServer: proxyServer,
tapServer: tapProxyServer,
}
}
func (s *yurtHubServer) Run() {
go func() {
err := s.hubServer.ListenAndServe()
if err != nil {
panic(err)
}
}()
go func() {
err := s.proxyServer.ListenAndServe()
if err != nil {
panic(err)
}
}()
klog.Infof("tap IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
err := s.tapServer.tapHttp.ListenAndServe()
if err != nil {
panic(err)
}
klog.Infof("tap1 IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
}