Docker安全及日志管理

DockerRemoteAPI访问控制
默认只开启了unix socket，如需开放http，做如下操作：
1、dockerd -H unix:///var/run/docker.sock -H tcp://192.168.180.210:2375
2、vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H unit:///var/run/docker.sock -H tcp://192.168.180.210:2375

vi /etc/sysconfig/docker
加上如下字段：
OPTIONS=‘–selinux-enabled -H unix://var/run/docker.sock -H tcp://0.0.0.0:2375’
//重启docker
systemctl daemon-reload
systemctl restart docker
netstat -nplt |grep 2375

(2)通过curl和http来获取docker的容器的相关信息
2.1 获取当前容器信息
通过curl：curl http://localhost:2375/containers/json

通过RemoteAPI获取docker的容器的相关信息
获取当前容器信息：
curl http://192.168.107.197:2375/containers/json
可以在浏览器中输入http://192.168.107.197:2375/containers/json
监控容器信息和导出容器
http://192.168.107.197:2375/containers/faf081fd4843/json
http://192.168.107.197:2375/containers/faf081fd4843/export
获取镜像相关信息
curl http://192.168.107.197:2375/images/json
http://192.168.107.197:2375/images/json

放行端口：
firewall-cmd --permanent --add-rich-rule=“rule family=“ipv4” source address=“192.168.107.197” port protocol=“tcp” port=“2375” accept”
firewall-reload

客户端访问：
docker -H=tcp://192.168.107.197:2375 ps

镜像的检验
CI=true dive

DockerClient 端与 DockerDaemon 的通信安全
yum install -y epel-release
yum install -y yum-utils device-mapper- persistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
sed -i ‘s/SELINUX=enforcing/SELINUX=disabled/’ /etc/selinux/config

docker master:
hostnamectl set-hostname master && bash
vim /etc/hosts
192.168.180.210 master
192.168.180.200 client
mkdir tls
cd tls
openssl genrsa -aes256 -out ca-key.pem 4096
openssl req -new -x509 -days 1000 -key ca-key.pem -sha256 -subj “/CN=master” -out ca.pem
openssl genrsa -out server-key.pem 4096
openssl req -subj “/CN=master” -sha256 -new -key server-key.pem -out server.csr
echo subjectAltName = DNS:master,IP:127.0.0.1 >> extfile.cnf
echo extendedKeyUsage = serverAuth >> extfile.cnf
openssl x509 -req -days 365 -sha256 -in server.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extfile extfile.cnf
openssl genrsa -out key.pem 4096
openssl req -subj ‘/CN=client’ -new -key key.pem -out client.csr
echo extendedKeyUsage = clientAuth > extfile-client.cnf
openssl x509 -req -days 365 -sha256 -in client.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile extfile-client.cnf
rm -rf client.csr server.csr extfile.cnf extfile-client.cnf ca.srl

vim /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd --tlsverify --tlscacert=/tls/ca.pem --tlscert=/tls/server-cert.pem --tlskey=/tls/server-key.pem -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock
systemctl daemon-reload
systemctl restart docker
scp ca.pem [email protected]:/etc/docker/
scp cert.pem [email protected]:/etc/docker/
scp key.pem [email protected]:/etc/docker/

客户端：
hostnamectl set-hostname client && bash
vim /etc/hosts
192.168.180.210 master
192.168.180.200 client
cd /etc/docker
docker --tlsverify --tlscacert=ca.pem --tlscert=cert.pem --tlskey=key.pem -H tcp://master:2376 version

export GODEBUG=x509ignoreCN=0

CPU 内存 硬盘

check created users

grep authorized_keys $dockerfile

check OS users

grep “etc/group” $dockerfile

Check sudo users

grep “etc/sudoers.d” $dockerfile

Check ssh key pair

grep “.ssh/.*id_rsa” $dockerfile

Add your checks in below…

git clone https://github.com/docker/docker-bench-security.git

stress
vim /root/stress/Dockerfile
FROM centos:7
MAINTAINER 5CC
RUN yum -y install wget
RUN wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
RUN wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
RUN yum -y install stress

docker build -t centos:stress .

docker run -tid --cpu-shares 100 centos:stress

docker run -tid --name cpu512 --cpu-shares 512 centos:stress stress -c 10
docker run -tid --name cpu1024 --cpu-shares 1024 centos:stress stress -c 10
docker run -tid --cpu-period 100000 --cpu-quota 200000 centos:stress
cat /sys/fs/cgroup/cpu/cpu.cfs_period_us
cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us

docker run -tid --name cpu1 --cpuset-cpus 0-2 centos:stress
cat /sys/fs/cgroup/cpuset/cpuset.cpus

docker exec 5204fe18208e taskset -c -p 1

docker run -tid --name cpu3 --cpuset-cpus 1 --cpu-shares 512 centos:stress stress -c 1

docker run -tid --name cpu4 --cpuset-cpus 0 --cpu-shares 1024 centos:stress stress -c 1

内存
docker run -it -m 200M --memory-swap=300M progrium/stress --vm 1 --vm-bytes 280M

docker run -it -m 200M --memory-swap=300M progrium/stress --vm 1 --vm-bytes 310M

BLKIO
docker run -it --name container_A --blkio-weight 600 centos:stress

docker run -it --device-write-bps /dev/sda:5MB centos:stress

ELK:

mkdir /var/log/Elasticsearch
chmod -R 777 /var/log/Elasticsearch

vim /etc/sysctl.conf
vm.max_map_count=655360
sysctl -p
vim /etc/security/limits.conf

• soft nofile 65535
• hard nofile 65535
• soft nproc 65535
• hard nproc 65535
• soft memlock unlimited
• hard memlock unlimited

docker network create ELK-kgc
dcoker network ls

cd /root/ELK/Elasticsearch

创建网络，并绑定网段
docker network create --driver bridge --subnet=172.19.0.0/16 ELK-kgc

运行Nginx:

复制nginx的配置文件到主机的/opt/nginx/目录下
配置日志格式：
log_format main '“$htt{p}_{u}se{r}_{a}gent""$http_x_forwarded_for” ’
'$remot{e}_{u}ser[$time_local] “KaTeX parse error: Double superscript at position 16: request" ' '̲status $bod{y}_{b}yte{s}_{s}ent"$http_referer” ’
‘$upstream_addr $upstream_status $upstream_response_time’;

构建nginx

docker run -itd -p 8011:80 --network ELK-kgc -v /var/log/nginx:/var/log/nginx -v /opt/nginx/html:/usr/share/nginx/html --name nginx-ELK --ip 172.19.0.200 nginx:kgc

在/opt/nginx/html目录下，创建index.html文件

创建日志文件：
vim /var/log/nginx/www.bdqn.cn-access.log
“YisouSpider”“106.11.155.156” - [18/Jul/2022:00:00:13 +0800] “GET /applier/position?gwid=17728&qyid=122257 HTTP/1.0” 200 9197 “-” 192.168.168.108:80 2000.032
“-”“162.209.213.146” - [18/Jul/2022:00:02:11 +0800] "GET //tag/7764.shtml HTTP/1.0"20024922 “-” 192.168.168.108:80 200 0.074
“YisouSpider”“106.11.152.248” - [18/Jul/2022:00:07:44 +0800] “GET /news/201712/21424.shtml HTTP/1.0” 200 8821 “-” 192.168.168.110:80 2000.097
“YisouSpider”“106.11.158.233” - [18/Jul/2022:00:07:44+0800]“GET/news/201301/7672.shtml HTTP/1.0” 200 8666 “-” 192.168.168.110:80 2000.111
“YisouSpider”“106.11.159.250” - [18/Jul/2022:00:07:44+0800]“GET/news/info/id/7312.html HTTP/1.0” 200 6617 “-” 192.168.168.110:80 2000.339
“Mozilla/5.0 (compatible;SemrushBot/2~bl;+http://www.semrush.com/bot.html)”“46.229.168.83”- [18/Jul/2022:00:08:57+0800]“GET/tag/1134.shtmlHTTP/1.0” 200 6030 “-” 192.168.168.108:80 200 0.079

运行Elasticsearch:
构建：
docker build -t elasticsearch .
运行：
docker run -itd --privileged -p 9200:9200 -p 9300:9300 --network ELK-kgc --ip 172.19.0.100 -v /var/log/elasticsearch:/var/log/elasticsearch --name elasticsearch elasticsearch
查看、验证：
curl -X PUT “localhost:9200/customer?pretty”
查看索引：
curl -X GET “localhost:9200/_cat/indices?v”

删除索引,通配符形式
curl -XDELETE localhost:9200/索引*

测试数据：
curl -H ‘Content-Type: application/x-ndjson’ -XPOST ‘localhost:9200/bank/account/_bulk?pretty’ --data-binary @accounts.json
curl -H ‘Content-Type: application/x-ndjson’ -XPOST ‘localhost:9200/shakespeare/doc/_bulk?pretty’ --data-binary @shakespeare_6.0.json
curl -H ‘Content-Type: application/x-ndjson’ -XPOST ‘localhost:9200/_bulk?pretty’ --data-binary @logs.jsonl

运行Kibana：
docker build -t kibana .
docker run -itd --privileged -p 5601:5601 --ip 172.19.0.110 --network ELK-kgc --name kibana kibana

运行logstash:
docker build -t logstash .
docker run -itd --privileged -p 5044:5044 --network ELK-kgc -v /opt/logstash/conf:/opt/logstash/conf --ip 172.19.0.120 --name logstash logstash

运行filebeat
docker build -t filebeat .
docker run -itd --privileged --network ELK-kgc -v /var/log/nginx:/var/log/nginx --ip 172.19.0.130 --name Filebeat filebeat

mkdir -p /opt/logstash/conf
#日志输入，可以是从stdin屏幕输入读取，可以从file指定的文件，也可以从es，filebeat，kafka，redis等读取
input {
beats {
port => 5044
}
}
#日志过滤，不是必须的
filter {
if “www-bdqn-cn-pro-access” in [tags] {
grok {
match => {“message” => ‘%{QS:agent} “%{IPORHOST:http_x_forwarded_for}” - [%{HTTPDATE:timestamp}]
“(?:%{WORD:verb}
%{NOTSPACE:request}(?: HTTP/%{NUMBER:http_version})?|-)” %{NUMBER:response}
%{NUMBER:bytes} %{QS:referrer} %{IPORHOST:remote_addr}:%{P
OSINT:port} %{NUMBER:remote_addr_response} %{BASE16FLOAT:request_time}’}
}
}
#用于解码被编码的字段,可以解决URL中 中文乱码的问题
urldecode {all_fields => true}
#日期解析字段 日期解析 解析字段中的日期，然后转存到@timestamp
date {
match => [ “timestamp” , “dd/MMM/YYYY:HH:mm:ss Z” ]
}
#添加有关用户代理(如系列,操作系统,版本和设备)的信息
useragent {
source =>“agent”
target =>“ua”
}
}
#输出字段 将事件发送到特定目标
output {
if “www-bdqn-cn-pro-access” in [tags]
{ Elasticsearch {
hosts => [“Elasticsearch:9200”]
manage_template => false
index =>“www-bdqn-cn-pro-access-%{+YYYY.MM.dd}”
}
}
}

filebeat
/var/lib/filebeat/registry

logstash
/usr/local/logstash-6.1.0/data/uuid

=ELKF日志收集=====================
一、安装docker:
yum install -y yum-utils device-mapperpersistent-data lvm2
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io
systemctl start docker && systemctl enable docker

修改内核参数
vim /etc/sysctl.conf
vm.max_map_count=655360
sysctl -p
vim /etc/security/limits.conf

• soft nofile 65535
• hard nofile 65535
• soft nproc 65535
• hard nproc 65535
• soft memlock unlimited
• hard memlock unlimited

二、构建Nginx
在opt目录下新建nginx目录
mkdir -p /opt/nginx/html
cd /opt/nginx
上传nginx的dockerfile
创建网络
docker network create --driver bridge --subnet=172.19.0.0/16 ELK-kgc
构建Nginx
docker build -t nginx .
运行nginx容器
docker run -itd -p 8011:80 --network ELK-kgc -v /var/log/nginx:/var/log/nginx -v /opt/nginx/html:/usr/share/nginx/html --name nginx-ELK --ip 172.19.0.200 nginx
在/opt/nginx/html目录下添加index.html文件

三、构建elasticsearch
在opt目录新建elasticsearch目录
mkdir /opt/elasticsearch
在/var/log/下新建elasticsearch目录，并赋予完全控制权限
mkdir /var/log/elasticsearch
chmod 777 /var/log/elasticsearch
cd /opt/elasticsearch/
上传dockerfile到/opt/elasticsearch目录
构建elasticsearch
docker build -t elasticsearch .
运行elasticsearch
docker run -itd -p 9200:9200 -p 9300:9300 --network ELK-kgc --ip 172.19.0.100 -v /var/log/elasticsearch:/var/log/elasticsearch --name elasticsearch elasticsearch
新建索引：
curl -X PUT “localhost:9200/customer?pretty”
查看索引：
curl -X GET “localhost:9200/_cat/indices?v”

四、构建kibana
在/opt/新建kibana目录
mkdir /opt/kibana
cd /opt/kibana
上传Dockerfile
构建Kibana：
docker build -t kibana .
运行：
docker run -itd -p 5601:5601 --ip 172.19.0.110 --network ELK-kgc --name kibana kibana

五、构建logstash
在opt目录下新建logstash目录
mkdir -p /opt/logstash/conf
cd /opt/logstash/
上传Dockerfile
mv nginx-log.conf conf
在/opt/logstash目录下新建conf,将nginx-log.conf拷贝到conf
构建：
docker build -t logstash .
运行：
docker run -itd -p 5044:5044 --network ELK-kgc -v /opt/logstash/conf:/opt/logstash/conf --ip 172.19.0.120 --name logstash logstash

六、构建filebeat
在opt新建filebeat目录
mkdir /opt/filebeat
cd /opt/filebeat
上传Dockerfile
构建：
docker build -t filebeat .
运行：
docker run -itd --network ELK-kgc -v /var/log/nginx:/var/log/nginx --ip 172.19.0.130 --name filebeat filebeat