【博客699】docker daemon预置iptables剖析
docker daemon预置iptables剖析
没有安装docker的机器：iptables为空，且每个链路的默认policy均为ACCEPT
[root@~]# iptables-save
[root@ ~]# iptables -t raw -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
[root@~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
[root@~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
[root@~]# iptables -t nat -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
安装了docker后的机器
[root@~]# iptables-save
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*raw
:PREROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*mangle
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*filter
:INPUT ACCEPT [0:0]
:FORWARD DROP [0:0]
:OUTPUT ACCEPT [0:0]
:DOCKER - [0:0]
:DOCKER-ISOLATION-STAGE-1 - [0:0]
:DOCKER-ISOLATION-STAGE-2 - [0:0]
:DOCKER-USER - [0:0]
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-ISOLATION-STAGE-1
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o docker0 -j DOCKER
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
-A DOCKER-USER -j RETURN
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
# Generated by iptables-save v1.8.4 on Mon Jul 10 18:12:12 2023
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A DOCKER -i docker0 -j RETURN
COMMIT
# Completed on Mon Jul 10 18:12:12 2023
[root@~]# iptables -t raw -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
[root@~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy DROP 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DOCKER-USER all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 DOCKER-ISOLATION-STAGE-1 all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- * docker0 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
0 0 DOCKER all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 docker0 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain DOCKER (1 references)
pkts bytes target prot opt in out source destination
Chain DOCKER-ISOLATION-STAGE-1 (1 references)
pkts bytes target prot opt in out source destination
0 0 DOCKER-ISOLATION-STAGE-2 all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-ISOLATION-STAGE-2 (1 references)
pkts bytes target prot opt in out source destination
0 0 DROP all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-USER (1 references)
pkts bytes target prot opt in out source destination
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
[root@~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
[root@~]# iptables -t nat -nvL
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
5 1121 DOCKER all -- * * 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 MASQUERADE all -- * !docker0 172.17.0.0/16 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
3 168 DOCKER all -- * * 0.0.0.0/0 !127.0.0.0/8 ADDRTYPE match dst-type LOCAL
Chain DOCKER (2 references)
pkts bytes target prot opt in out source destination
0 0 RETURN all -- docker0 * 0.0.0.0/0 0.0.0.0/0
docker基础的iptables网络要实现以下目标
1、docker网络与宿主机网络分开：
在机器上下发一条iptables，实际只可以拦截宿主机其他进程，拦截不了docker进程
iptables -P INPUT DROP
-
所谓的 Docker 网络的隔离性只在 INPUT 链，OUTPUT 链中体现。修改 PREROUTING 链，FORWARD 链，POSTROUTING 链都会影响到 Docker 容器的网络环境。要通过 DOCKER 链控制 Docker 容器的访问权限，需要先删除 -A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 这条规则, 因为这条规则的不确定性太大。然后单纯通过 DOCKER 链来控制 Docker 容器的网络访问权限.
-
容器的请求没有经过宿主机的 INPUT 链，所以在宿主机的 INPUT 链上做规则是没法限制容器的网络访问的. 需要限制容器网络访问应该对 DOCKER 链动手脚。
2、docker网络里不同docker桥之间网络不互通
3、将FORWARD链的默认规则改为drop，原因如下：
- 当 docker 启动时，它会启用net.ipv4.ip_forward而不将 iptablesFORWARD链默认策略更改为DROP。这意味着与 docker 主机位于同一网络上的另一台计算机可以将路由添加到其路由表中，并直接寻址该 docker 主机上运行的任何容器。
issue：https://github.com/moby/moby/issues/14041 - 要转发流量需要net.ipv4.ip_forward，而net.ipv4.ip_forward默认是关的
- net.ipv4.ip_forward 只要在docker daemon 启动之前手动把这个参数设置为1，docker daemon 默认就认为有人用iptables的filter表的forward链了，就不会更改forward链的默认策略了。
docker iptables链剖析
docker的几条链路：
-
Docker的DOCKER链：仅处理从宿主机到docker0的IP数据包。
-
Docker的DOCKER-ISOLATION链：为了隔离在不同的bridge网络之间的容器，Docker提供了两个DOCKER-ISOLATION阶段实现。
- DOCKER-ISOLATION-STAGE-1链过滤源地址是bridge网络（默认docker0）的IP数据包，匹配的IP数据包再进入DOCKER-ISOLATION-STAGE-2链处理，不匹配就返回到父链FORWARD。
- 在DOCKER-ISOLATION-STAGE-2链中，进一步处理目的地址是bridge网络的IP数据包，匹配的IP数据包表示该IP数据包是从一个bridge网络的网桥发出，
到另一个bridge网络的网桥，这样的IP数据包来自其他bridge网络，将被直接DROP；不匹配的IP数据包就返回到父链FORWARD继续进行后续处理。
-
Docker的DOCKER-USER链
用于用户可以自定义添加自己对容器的处理规则
注意：
-
Docker启动时，会加载DOCKER链和DOCKER-ISOLATION（现在是DOCKER-ISOLATION-STAGE-1）链中的过滤规则，并使之生效，绝对禁止修改这里的过滤规则。
-
如果用户要补充Docker的过滤规则，强烈建议追加到DOCKER-USER链。DOCKER-USER链中的过滤规则，将先于Docker默认创建的规则被加载，从而能够覆盖Docker在DOCKER链和DOCKER-ISOLATION链中的默认过滤规则。例如，Docker启动后，默认任何外部source IP都被允许转发，从而能够从该source IP连接到宿主机上的任何Docker容器实例。如果只允许一个指定的IP访问容器实例，可以插入路由规则到DOCKER-USER链中，从而能够在DOCKER链之前被加载。
iptables规则分析：
1、DOCKER链：对入包和出包都拦截到自定义Docker链路
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A DOCKER -i docker0 -j RETURN
2、DOCKER-ISOLATION链：
默认只有docker0，此时可能不好理解，对于源是docker0，目的不是docker0，且目的是docker0的
流量进行丢弃，看起来好像矛盾，因为对于入向是docker0，出向不是docker0的流量跳转到
DOCKER-ISOLATION-STAGE-2时，必定不会再匹配上了，但是其实是因为默认值只有docker0，
默认情况下DOCKER-ISOLATION-STAGE-2的drop不会命中，因为只有一个docker网桥
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
当再添加一个网桥的时候，docker network create -d bridge docker1，此时就变成了：
-A DOCKER-ISOLATION-STAGE-1 -i br-fc313d59a28d ! -o br-fc313d59a28d -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -i docker0 ! -o docker0 -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
-A DOCKER-ISOLATION-STAGE-2 -o br-fc313d59a28d -j DROP
-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
这时候就解释通了，配合起来就是：
源是br-fc313d59a28d，目的不是br-fc313d59a28d，但目的是docker0的会被drop，
源是docker0，目的不是docker0，但目的是br-fc313d59a28d的会被drop。
也就是说-A DOCKER-ISOLATION-STAGE-2 -o docker0 -j DROP是为了去拦截非docker0的
那些网桥来访问docker0，而-A DOCKER-ISOLATION-STAGE-2 -o br-fc313d59a28d -j DROP
是为了去拦截非br-fc313d59a28d的那些网桥来访问br-fc313d59a28d
DOCKER-USER链：存放用户自己的链
隔离规则再剖析：
再创建一个docker网络桥：可以看到在隔离的规则除了原来的docker0，同时也为docker0对应的网桥br-fc313d59a28d也多了限制规则
docker network create -d bridge docker1
[root@10-234-37-2 ~]# iptables -t filter -nvL
Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy DROP 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
0 0 DOCKER-USER all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 DOCKER-ISOLATION-STAGE-1 all -- * * 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- * br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
0 0 DOCKER all -- * br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- br-fc313d59a28d !br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- br-fc313d59a28d br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- * docker0 0.0.0.0/0 0.0.0.0/0 ctstate RELATED,ESTABLISHED
0 0 DOCKER all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 ACCEPT all -- docker0 docker0 0.0.0.0/0 0.0.0.0/0
Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain DOCKER (2 references)
pkts bytes target prot opt in out source destination
Chain DOCKER-ISOLATION-STAGE-1 (1 references)
pkts bytes target prot opt in out source destination
0 0 DOCKER-ISOLATION-STAGE-2 all -- br-fc313d59a28d !br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0
0 0 DOCKER-ISOLATION-STAGE-2 all -- docker0 !docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-ISOLATION-STAGE-2 (2 references)
pkts bytes target prot opt in out source destination
0 0 DROP all -- * br-fc313d59a28d 0.0.0.0/0 0.0.0.0/0
0 0 DROP all -- * docker0 0.0.0.0/0 0.0.0.0/0
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0
Chain DOCKER-USER (1 references)
pkts bytes target prot opt in out source destination
0 0 RETURN all -- * * 0.0.0.0/0 0.0.0.0/0