Shell集群部署ELK+Zookeeper+Kafka+filebeat

服务器 2核4G

filebat->kafka->logstash->elasticsearch->kibana

Elasticsearch

Node-1

#!/bin/bash
# 用户/密码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf < /etc/sysctl.conf < /etc/security/limits.conf </dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [ ! -f /tmp/$file ]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创建目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到/usr/local/es/config/certs/
将http.p12和elastic-certificates.p12上传到/usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-1
# ES默认 只允许本地 127.0.0.1 和[::1] 访问
# 也可以设置成0.0.0.0 允许所有IP主机访问
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入警告但是能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会升级为异常,ES无法正确启动。
network.host: 0.0.0.0
# 默认ES节点端口9200，如果是伪集群（在一台服务器上搭建集群），需要修改。
http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否使用内存交换分区
bootstrap.memory_lock: false
# 设置数据存放路径
path.data: /usr/local/es/data/
# 设置日志存放路径
path.logs: /usr/local/es/logs/
# 发现其他节点主机配置 这里配置的是ES所在服务器的公网IP地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点可以被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#增加新的参数，为了让elasticsearch-head插件可以访问es (5.x版本，如果没有可以自己手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
    #  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此进程可以打开的最大文件数
LimitNOFILE=65535
# 指定此进程可以打开的最大进程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0-永不超时
TimeoutStopSec=0
# SIGTERM是停止java进程的信号
KillSignal=SIGTERM
# 信号只发送给给JVM
KillMode=process
# java进程不会被杀掉
SendSIGKILL=no
# 正常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es

Node-2

#!/bin/bash
# 用户/密码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf < /etc/sysctl.conf < /etc/security/limits.conf </dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [ ! -f /tmp/$file ]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创建目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到/usr/local/es/config/certs/
将http.p12和elastic-certificates.p12上传到/usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-2
# ES默认 只允许本地 127.0.0.1 和[::1] 访问
# 也可以设置成0.0.0.0 允许所有IP主机访问
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入警告但是能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会升级为异常,ES无法正确启动。
network.host: 0.0.0.0
# 默认ES节点端口9200，如果是伪集群（在一台服务器上搭建集群），需要修改。
http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否使用内存交换分区
bootstrap.memory_lock: false
# 设置数据存放路径
path.data: /usr/local/es/data/
# 设置日志存放路径
path.logs: /usr/local/es/logs/
# 发现其他节点主机配置 这里配置的是ES所在服务器的公网IP地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点可以被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#增加新的参数，为了让elasticsearch-head插件可以访问es (5.x版本，如果没有可以自己手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
#  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此进程可以打开的最大文件数
LimitNOFILE=65535
# 指定此进程可以打开的最大进程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0-永不超时
TimeoutStopSec=0
# SIGTERM是停止java进程的信号
KillSignal=SIGTERM
# 信号只发送给给JVM
KillMode=process
# java进程不会被杀掉
SendSIGKILL=no
# 正常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es

Node-3

#!/bin/bash
# 用户/密码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf < /etc/sysctl.conf < /etc/security/limits.conf </dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [ ! -f /tmp/$file ]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创建目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到/usr/local/es/config/certs/
将http.p12和elastic-certificates.p12上传到/usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-3
# ES默认 只允许本地 127.0.0.1 和[::1] 访问
# 也可以设置成0.0.0.0 允许所有IP主机访问
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入警告但是能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会升级为异常,ES无法正确启动。
network.host: 0.0.0.0
# 默认ES节点端口9200，如果是伪集群（在一台服务器上搭建集群），需要修改。
http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否使用内存交换分区
bootstrap.memory_lock: false
# 设置数据存放路径
path.data: /usr/local/es/data/
# 设置日志存放路径
path.logs: /usr/local/es/logs/
# 发现其他节点主机配置 这里配置的是ES所在服务器的公网IP地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点可以被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#增加新的参数，为了让elasticsearch-head插件可以访问es (5.x版本，如果没有可以自己手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
#  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此进程可以打开的最大文件数
LimitNOFILE=65535
# 指定此进程可以打开的最大进程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0-永不超时
TimeoutStopSec=0
# SIGTERM是停止java进程的信号
KillSignal=SIGTERM
# 信号只发送给给JVM
KillMode=process
# java进程不会被杀掉
SendSIGKILL=no
# 正常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es