graylog集群(es+mongo+graylog-sidecar+nginx负载均衡) 全docker配置（3节点）

一、资源列表与拓扑

注：初始环境等跳过配置

node1	192.168.1.100	mongo:latest	elasticsearch-oss:7.10.2	graylog:4.3
node2	192.168.1.101	mongo:latest	elasticsearch-oss:7.10.2	graylog:4.3
node3	192.168.1.102	mongo:latest	elasticsearch-oss:7.10.2	graylog:4.3

 

二、mongo-service （replSet）集群配置

####可以直接3台全部pull
docker pull mongo
####如内网机器，需先在一台上pull后，将镜像保存，在其它2台load
docker save mongo:latest -o /data/mongo.tar.gz
scp -r /data/mongo.tar.gz [email protected]:/data/
scp -r /data/mongo.tar.gz [email protected]:/data/
docker load -i /data/mongo.tar.gz

####创建挂载目录
mkdir /data/mongo/{data,config}
####启动命令
docker run -itd --restart=always --name mongo-service -p 27017:27017 -v /data/mongo/data:/data/db -v /data/mongo/config:/data/configdb mongo:latest --replSet colonyReplSet

####选择任一配置集群，node1
docker exec -it mongo-service /bin/bash
mongo
config={
"_id" : "colonyReplSet",
"version" : 1,
"members" : [
{
"_id" : 0,
"host" : "192.168.1.100:27017",
"priority" : 6
},
{
"_id" : 1,
"host" : "192.168.1.101:27017",
"priority" : 3
},
{
"_id" : 2,
"host" : "192.168.1.102:27017",
"priority" : 2
}
]
}
rs.initiate(config)

####选择任一查看状态
rs.status()

三、elasticsearch-service 集群配置

####3台
docker pull elasticsearch:7.10.2

####如内网机器，需先在一台上pull后，将镜像保存，在其它2台load
docker save docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 -o /data/es-docker-7.10.2.tar.gz
scp -r es-docker-7.10.2.tar.gz  [email protected]:/data/
scp -r es-docker-7.10.2.tar.gz  [email protected]:/data/
docker load -i /data/es-docker-7.10.2.tar.gz

####创建挂载目录
mkdir /data/elasticsearch/config -p
mkdir /data/elasticsearch/config/{data,logs,plugins}

####node1
vim /data/elasticsearch/config/es.yml
cluster.name: elasticsearch-cluster
cluster.initial_master_nodes : es-node1
node.name: es-node1
network.bind_host: 0.0.0.0
network.publish_host: 192.168.1.100
http.port: 9201
transport.tcp.port: 9301
discovery.zen.minimum_master_nodes: 2
http.max_content_length: 2000mb
http.max_header_size: 1024k
http.max_initial_line_length: 1024k
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true  
discovery.zen.ping.unicast.hosts:  ["192.168.1.100:9301","192.168.1.101:9301","192.168.1.102:9301"]

####node2
vim /data/elasticsearch/config/es.yml
cluster.name: elasticsearch-cluster   #集群名（三个节点一致）
node.name: es-node2   #节点名（每台节点不同）
network.bind_host: 0.0.0.0
network.publish_host: 192.168.1.101  #对外地址
http.port: 9201 #对外端口（每台机器装一个ES则不用改）
transport.tcp.port: 9301 #ES内部通信端口（每台机器装一个ES则不用改）
discovery.zen.minimum_master_nodes: 2 #主节点数/2+1,防止脑裂
http.max_content_length: 2000mb
http.max_header_size: 1024k
http.max_initial_line_length: 1024k
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: false
node.data: true  
discovery.zen.ping.unicast.hosts:  ["192.168.1.100:9301","192.168.1.101:9301","192.168.1.102:9301"]

####node3
vim /data/elasticsearch/config/es.yml
cluster.name: elasticsearch-cluster   #集群名（三个节点一致）
node.name: es-node2   #节点名（每台节点不同）
network.bind_host: 0.0.0.0
network.publish_host: 192.168.1.102  #对外地址
http.port: 9201 #对外端口（每台机器装一个ES则不用改）
transport.tcp.port: 9301 #ES内部通信端口（每台机器装一个ES则不用改）
discovery.zen.minimum_master_nodes: 2 #主节点数/2+1,防止脑裂
http.max_content_length: 2000mb
http.max_header_size: 1024k
http.max_initial_line_length: 1024k
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: false
node.data: true  
discovery.zen.ping.unicast.hosts:  ["192.168.1.100:9301","192.168.1.101:9301","192.168.1.102:9301"]

####3台配置目录权限
chmod 777 /data/elasticsearch/config -R

####启动docker
docker run -itd -e ES_JAVA_OPTS="-Xms8g -Xmx8g -Dlog4j2.formatMsgNoLookups=true" \
-p 9201:9201 -p 9301:9301 \
-e ES_MIN_MEM=1024m \
-e ES_MAX_MEM=8192m \
-v /data/elasticsearch/config/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v /data/elasticsearch/config/data/:/usr/share/elasticsearch/data/ \
-v /data/elasticsearch/config/logs/:/usr/share/elasticsearch/logs/ \
--restart=on-failure \
--name ES \
docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2

##查看状态

[root@graylog-master config]# curl http://127.0.0.1:9201/_cat/nodes?pretty
192.168.1.102 2 75 1 0.37 0.45 0.27 dir  - es-node2
192.168.1.101 0 75 1 0.28 0.21 0.13 dir  - es-node2
192.168.1.100 1 75 0 0.17 0.34 0.40 dimr * es-node1

四、graylog 集群配置

####3台直接pull
docker pull graylog/graylog:4.3

####如内网机器，需先在一台上pull后，将镜像保存，在其它2台load
docker save graylog/graylog:4.3 -o /data/graylog-4.3.tar.gz
docker scp -r /data/graylog-4.3.tar.gz [email protected]:/data/
docker scp -r /data/graylog-4.3.tar.gz [email protected]:/data/
docker load -i /data/graylog-4.3.tar.gz

####创建挂载目录
mkdir /data/graylog/ -p

####安装sha
yum install perl-Digest-SHA pwgen -y
pwgen -N 1 -s 96
echo -n admin| shasum -a 256

####node1上配置文件
cat graylog.conf
is_master = true
node_id_file = /usr/share/graylog/data/config/node-id
password_secret =
root_password_sha2 =
root_timezone = Asia/Shanghai
bin_dir = /usr/share/graylog/bin
data_dir = /usr/share/graylog/data
plugin_dir = /usr/share/graylog/plugin
http_bind_address = 0.0.0.0:9000
http_publish_uri = http://192.168.1.100:9000/
elasticsearch_hosts = http://192.168.1.100:9201,http://192.168.1.101:9201,http://192.168.1.102:9201
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 4
elasticsearch_replicas = 0
elasticsearch_index_prefix = graylog
allow_leading_wildcard_searches = false
allow_highlighting = true
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 8
outputbuffer_processors = 16
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = data/journal
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://192.168.1.100:27017/graylog,192.168.1.101:27017/graylog,192.168.1.102:27017/graylog
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
proxied_requests_thread_pool_size = 32 

####node2 node3(修改以下2行)
is_master = false
http_publish_uri = http://192.168.1.10????:9000/


##启动前配置，经测试 如无权限相关配置，docker启动失败
mkdir /data/graylog/journal -p
mkdir /data/graylog/data/config -p
将如上graylog.conf 放到config目录下
chmod 777 /data/graylog -R 

#master启动命令（其它2台更改ip即可）
docker run -itd \
--link es \
--link mongo-service \
-p 9000:9000 \
-p 5044:5044 \
-p 12201:12201 -p 12201:12201/udp \
-p 5140:5140 -p 5140:5140/udp \
-p 5555:5555 -p 5555:5555/udp \
-p 10000:10000 -p 10000:10000/udp \
-p 13301:13301 -p 13301:13301/udp \
-p 13302:13302 -p 13302:13302/udp \
-e GRAYLOG_HTTP_EXTERNAL_URI=http://192.168.1.100:9000/ \
-e GRAYLOG_PASSWORD_SECRET=somepasswordpepper \
-e GRAYLOG_ROOT_PASSWORD_SHA2=8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 \
-v /etc/localtime:/etc/localtime:ro \
-v /data/graylog/journal:/usr/share/graylog/data/journal \
-v /data/graylog/data/config:/usr/share/graylog/data/config \
--name graylog \
graylog/graylog:4.2

 4.1登陆graylog查看system/nodes节点

4.2graylog-sidecar收集日志

创建收集日志的Inputs 建立输入：system/inputs/（select input-> Beats  ）/ Launch new input  /(配置如下 主题&端口 其它默认即可) / save （集群必须勾选Global）

4.3创建一个api token 客户端配置的时候使用

 System/sidecar -> Create or reuse a token for the graylog-sidecar user.

记住 token，配置客户端使用 

4.4客户端配置graylog-sidecar收集日志

sudo rpm -ivh https://packages.graylog2.org/repo/el/sidecar-stable/1.2/x86_64/graylog-sidecar-1.2.0-1.x86_64.rpm

cat /etc/graylog/sidecar/sidecar.yml
server_url: "http://192.168.1.100:9000/api/"
server_api_token: "8dbf0i0l6q2gmgul0j3ki552l68tphi22vbhu0fcrrpbc7e549" ###token填写自己的
node_id: "yoyiweb"  ###自定义，或用主机名
update_interval: 10
tls_skip_verify: false
send_status: true

graylog-sidecar -service install
systemctl start  graylog-sidecar
systemctl enable  graylog-sidecar
systemctl status  graylog-sidecar

4.5服务端配置sidecar 

在System/sidecar → configuration→ create configuration ->输入名字、选择collector(filebeat on Linux)→修改config里面的信息（paths 及 output地址）

 

  配置关联 System/sidecar →Administration->选中chemex下的filebeat->点击右侧configure→ 关联选中yoyi-web

 客户端配置查看服务，会自动启动一个filebeat进程，传输日志

4.6问题1： 收集处理单节点问题，如图显示

解决措施:

办法1：在客户端配置时，ip填写注意将各节点均分；

局限性：当节点挂掉之后，日志收集不上来，更改ip等需要更改配置文件，重启graylog-sidear 服务

办法2：netscaler创建vip，客户端填写vip

局限性：需要采购硬件slb

办法3：与2相同，创建一个nginx 负载均衡，进行转发

局限性：增大了运维难度

五、nginx 负载均衡 docker部署

docker pull nginx
mkdir /data/nginx  -p

##配置文件，添加负载
##8080--》graylog web
##8081--》graylog input 12201
##8082--》graylog input 12202
##8083--》graylog input 12203

cat nginx.conf 
user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

#    include /etc/nginx/conf.d/*.conf;
}

stream {
    upstream graylog_servers{
      server 192.168.1.100:9000;
      server 192.168.1.101:9000;
      server 192.168.1.102:9000;
    }
    server {
        listen       9000;
        proxy_pass   graylog_servers;
    }

    upstream server_input1{
	server 192.168.1.100:12201;
	server 192.168.1.101:12201;
	server 192.168.1.102:12201;
    }
    server {
	listen       12201;    
	proxy_pass  server_input1;
    }	

    upstream server_input2{
	server 192.168.1.100:12202;
	server 192.168.1.101:12202;
	server 192.168.1.102:12202;
    }
    server {
	listen       12202;
	proxy_pass  server_input2;
    }

    upstream server_input3{
	server 192.168.1.100:12203;
	server 192.168.1.101:12203;
	server 192.168.1.102:12203;
    }
    server {
	listen       12203; 
        proxy_pass  server_input3;
    }
}


##docker 启动
docker run -itd \
--link es \
--link mongo-service \
--link graylog \
-p 8080:9000 \
-p 8081:12201 \
-p 8082:12202 \
-p 8083:12203 \
-v /etc/localtime:/etc/localtime:ro \
-v /data/nginx/nginx.conf:/etc/nginx/nginx.conf \
--name nginx \
nginx

5.1 nginx 负载完后，需修改对应配置文件

需修改客户端gaylog-sideater 中url地址

 cat /etc/graylog/sidecar/sidecar.yml
server_url: "http://192.168.1.100:8080/api/"

5.2需修改服务器System/sidecar → configuration 中信息

output.logstash:
   hosts: ["192.168.1.100:8081"]

5.3查看各node状态