Naxsi + Nginx + Nxapi + ElasticSearch + Kibana 安装

Platform: CentOS 7 -- Linux localhost.localdomain 3.10.0-693.21.1.el7.x86_64 #1 SMP Wed Mar 7 19:03:37 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

安装依赖

---

• 安装 libssl: yum install openssl-devel.x86_64
• 安装 libpcre: yum install pcre-devel.x86_64
• 安装 zlib （系统默认已安装）
• 安装 gzip （系统默认已安装）

安装 Nginx + Naxsi

---

• 下载 Nginx: wget http://nginx.org/download/nginx-1.13.12.tar.gz
• 解压缩: tar -vzxf nginx-1.13.12.tar.gz
• 下载 Naxsi: git clone https://github.com/nbs-system/naxsi.git
• cd nginx-1.13.12, 编译

 ./configure --conf-path=/etc/nginx/nginx.conf \
 --add-module=../naxsi/naxsi_src/ \
 --error-log-path=/var/log/nginx/error.log \
 --http-client-body-temp-path=/var/lib/nginx/body \
 --http-fastcgi-temp-path=/var/lib/nginx/fastcgi \
 --http-log-path=/var/log/nginx/access.log \
 --http-proxy-temp-path=/var/lib/nginx/proxy \
 --lock-path=/var/lock/nginx.lock \
 --pid-path=/var/run/nginx.pid \
 --with-http_ssl_module \
 --with-http_v2_module \
 --with-http_gzip_static_module \
 --with-http_realip_module \
 --with-http_flv_module \
 --with-http_mp4_module \
 --without-mail_pop3_module \
 --without-mail_smtp_module \
 --without-mail_imap_module \
 --without-http_uwsgi_module \
 --without-http_scgi_module \

make && make install

• 安装完成

[root@localhost local]# nginx -v
nginx version: nginx/1.13.12

• 查看 nginx 安装位置

[root@localhost nginx]# whereis nginx
nginx: /usr/sbin/nginx /etc/nginx

• 编写启动脚本 /etc/init.d/nginx

#!/bin/bash
# nginx Startup script for the Nginx HTTP Server
# it is v.0.0.2 version.
# chkconfig: - 85 15
# description: Nginx is a high-performance web and proxy server.
#       It has a lot of features, but it's not for everyone.
# processname: nginx
# pidfile: /run/nginx.pid
# config: /etc/nginx/nginx.conf  -> 根据具体安装位置填写

# 根据具体安装位置填写
nginxd=/usr/sbin/nginx

# config
nginx_config=/etc/nginx/nginx.conf

# CentOS PID 文件放在 /run/ 下
nginx_pid==/run/nginx.pid

RETVAL=0
prog="nginx"
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "${NETWORKING}" = "no" ] && exit 0
[ -x $nginxd ] || exit 0
# Start nginx daemons functions.
start() {
if [ -e $nginx_pid ];then
  echo "nginx already running...."
  exit 1
fi
  echo -n $"Starting $prog: "
  daemon $nginxd -c ${nginx_config}
  RETVAL=$?
  echo
  [ $RETVAL = 0 ] && touch /var/lock/subsys/nginx
  return $RETVAL
}
# Stop nginx daemons functions.
stop() {
    echo -n $"Stopping $prog: "
    killproc $nginxd
    RETVAL=$?
    echo

    # 注意修改 PID 文件位置
    [ $RETVAL = 0 ] && rm -f /var/lock/subsys/nginx /run/nginx.pid
}
# reload nginx service functions.
reload() {
  echo -n $"Reloading $prog: "
  #kill -HUP `cat ${nginx_pid}`
  killproc $nginxd -HUP
  RETVAL=$?
  echo
}
# See how we were called.
case "$1" in
start)
    start
    ;;
stop)
    stop
    ;;
reload)
    reload
    ;;
restart)
    stop
    start
    ;;
status)
    status $prog
    RETVAL=$?
    ;;
*)
    echo $"Usage: $prog {start|stop|restart|reload|status|help}"
    exit 1
esac
exit $RETVAL

• 设置执行权限: chmod a+x /etc/init.d/nginx
• 注册成服务: chkconfig --add nginx
• 启动/关闭 nginx 服务

[root@localhost ~]# /etc/init.d/nginx start
Starting nginx (via systemctl):  [  OK  ]
[root@localhost ~]# ps aux | grep nginx
root     18150  0.0  0.0  46272  1136 ?        Ss   19:01   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nobody   18151  0.0  0.0  46708  1924 ?        S    19:01   0:00 nginx: worker process
root     18167  0.0  0.0 112660   976 pts/1    S+   19:01   0:00 grep --color=auto nginx

[root@localhost ~]# /etc/init.d/nginx stop
Stopping nginx (via systemctl):  [  OK  ]
[root@localhost ~]# ps aux | grep nginx
root     18195  0.0  0.0 112660   972 pts/1    S+   19:01   0:00 grep --color=auto nginx

[root@localhost ~]# service nginx start
Starting nginx (via systemctl):  [  确定  ]
[root@localhost ~]# ps aux | grep nginx
root     18234  0.0  0.0  46272  1124 ?        Ss   19:02   0:00 nginx: master process /usr/sbin/nginx -c /etc/nginx/nginx.conf
nobody   18235  0.6  0.0  46708  1916 ?        S    19:02   0:00 nginx: worker process
root     18238  0.0  0.0 112660   972 pts/1    S+   19:02   0:00 grep --color=auto nginx

[root@localhost ~]# service nginx stop
Stopping nginx (via systemctl):  [  确定  ]
[root@localhost ~]# ps aux | grep nginx
root     18271  0.0  0.0 112660   976 pts/1    S+   19:02   0:00 grep --color=auto nginx

Nginx + Naxsi 基本配置

---

• Naxsi 目录下的 naxsi_core.rules 拷贝至 nginx.conf 所在目录

cp /usr/local/naxsi/naxsi_config/naxsi_core.rules /etc/nginx/naxsi_core.rules

• 配置 nginx.conf 在 http 部分，引用 naxsi_core.rules

http {
    inclued       naxsi_core.rules;
    ...
}

• 配置 server 部分，开启 naxsi

location / {
   #开启 naxsi
   SecRulesEnabled;
   #开启学习模式的location，请求不能被block
   #LearningMode;
   #定义阻止请求的位置
   DeniedUrl "/50x.html"; 
   #CheckRules, 确定 naxsi 何时采取行动
   CheckRule "$SQL >= 8" BLOCK;
   CheckRule "$RFI >= 8" BLOCK;
   CheckRule "$TRAVERSAL >= 4" BLOCK;
   CheckRule "$EVADE >= 4" BLOCK;
   CheckRule "$XSS >= 8" BLOCK;
   #naxsi 日志文件
   error_log /var/log/nginx/security.log;
   ...
  }

• 重启 nginx: service nginx restart
• 关闭 CentOS 的 防火墙: systemctl stop firewalld
• 访问 http://server.ip:server.port/?test=<>
  
  http://server.ip:server.port/?test=<>
• 查看 /var/log/nginx/security.log

2018/05/07 20:08:47 [error] 18671#0: *30 NAXSI_FMT: ip=xxx.xxx.xxx.xxx&server=172.16.2.17&uri=/&learning=0&vers=0.56&total_processed=2&total_blocked=2&block=1&cscore0=$XSS&score0=8&zone0=ARGS&id0=1302&var_name0=test, client: xxx.xxx.xxx.xxx, server: , request: "GET /?test=%3C%3E HTTP/1.1", host: "172.16.2.17"

安装 ElasticSearch

---

• 安装依赖 java 1.8: yum install java-1.8.0-openjdk.x86_64

[root@localhost ~]# java -version
openjdk version "1.8.0_161"
OpenJDK Runtime Environment (build 1.8.0_161-b14)
OpenJDK 64-Bit Server VM (build 25.161-b14, mixed mode)

• 从官网下载 elasticsearch 安装包 (到目前5/9/2018，nxapi支持的版本为 ES 5.4.0, 比这个新的版本会报错): wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.0.tar.gz
• 解压缩: tar -vxf elasticsearch-5.4.0.tar.gz
• ElasticSearch 不能使用 root 用户运行，因此建立新的用户和组

groupadd elasticsearch
useradd -g elasticsearch elasticsearch
chown -R elasticsearch:elasticsearch elasticsearch-5.4.0

[root@localhost local]# ls -l
总用量 29372
drwxr-xr-x. 2 root          root                 6 11月  5 2016 bin
drwxr-xr-x. 9 elasticsearch elasticsearch      155 5月   8 14:51 elasticsearch-5.4.0
...

• 用户切换到elasticsearch: su elasticsearch
• 修改 elasticsearch 配置文件:

[root@localhost local]# vi elasticsearch-5.4.0/config/elasticsearch.yml
# 修改 IP 地址
network.host: 127.0.0.1
# 修改 port
http.port: 9200

• cd elasticsearch-5.4.0/bin，运行./elasticsearch, 产生 ERROR

ERROR: [2] bootstrap checks failed
[1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]
[2]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]

• 问题[2]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]虚拟内存太小
• 解决问题[2]: 用户切换到root，修改/etc/sysctl.conf

[root@localhost bin]# vi /etc/sysctl.conf
vm.max_map_count=262144

• 执行sysctl -p, 用户切换到elasticsearch, 重新执行./elasticsearch， 问题[2]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]解决

ERROR: [1] bootstrap checks failed
[1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]

• 问题 [1]: max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]最大可创建文件数量太少
• 解决问题[1]: 用户切换到root，修改/etc/security/limit.conf

[root@localhost bin]# vi /etc/security/limits.conf 
*               soft    nofile          65536
*               hard    nofile          65536

• 重启，用户切换到elasticsearch，重新运行./elasticsearch -d, 问题[1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]解决，成功运行 ES
• 访问localhost:9200

[root@localhost ~]# curl "localhost:9200"
{
  "name" : "H_vcJvG",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "d-jMp3O9Szq7nCQ7HHat2w",
  "version" : {
    "number" : "5.4.0",
    "build_hash" : "780f8c4",
    "build_date" : "2017-04-28T17:43:27.229Z",
    "build_snapshot" : false,
    "lucene_version" : "6.5.0"
  },
  "tagline" : "You Know, for Search"
}

• Elasticsearch 安装成功

• 在 ES 中添加 nxapi index: curl -XPUT 'http://localhost:9200/nxapi/'

安装 NXAPI

---

• 安装 elasticsearch5 模块: pip install elasticsearch5
  Python Elasticsearch 模块默认为版本6，为防止出现兼容性问题， 选择安装 elasticsearch5 模块: elasticsearch5 (5.5.2) - Python client for Elasticsearch

• 修改 nxapi 程序中的模块引入部分

# 当前目录
[root@localhost nxapi]# pwd
/usr/local/naxsi/nxapi

# 修改 nxtool.py
# 将 import elasticsearch 修改为 import elasticsearch5 as elasticsearch
[root@localhost nxapi]# vi nxtool.py
import elasticsearch5 as elasticsearch

# 修改 nxparse.py
# 将 from elasticsearch.helpers import bulk 修改为 from elasticsearch5.helpers import bulk
[root@localhost nxapi]# vi nxapi/nxparse.py
from elasticsearch5.helpers import bulk

# 修改 nxtypificator.py
# 将 from elasticsearch import Elasticsearch 修改为 from elasticsearch5 import Elasticsearch
[root@localhost nxapi]# vi nxapi/nxtypificator.py
from elasticsearch5 import Elasticsearch

• 在/usr/local/naxsi/nxapi目录下, 安装 nxapi

python setup.py build
python setup.py install

• 查看安装位置

[root@localhost nxapi]# whereis nxtool.py
nxtool: /usr/bin/nxtool.py
[root@localhost nxapi]# whereis nxapi.json
nxapi: /usr/local/etc/nxapi.json /usr/local/nxapi

• 向 ES 中导入 naxsi log：nxtool.py -c /usr/local/etc/nxapi.json --files=/var/log/nginx/security.log, 提示缺少 GeoIP 模块

• 安装 python GeoIP 模块

yum install python-devel GeoIP-devel 
pip install geoip

• 重新向 ES 中导入 naxsi log：nxtool.py -c /usr/local/etc/nxapi.json --files=/var/log/nginx/security.log，错误提示

[root@localhost ~]# nxtool.py -c /usr/local/etc/nxapi.json --file=/var/log/nginx/security.log 
# size :1000
ERROR:root:Unable to load GeoIPdb.
Unable to get GeoIP

• 修改/usr/local/etc/nxapi.json "naxsi" 部分

 "naxsi" : {
     "rules_path" : "/etc/nginx/naxsi_core.rules",
     "template_path" : [ "/usr/local/nxapi/tpl/"],
     "geoipdb_path" : "/usr/local/nxapi/country2coords.txt"
     },

• 重新向 ES 中导入 naxsi log：nxtool.py -c /usr/local/etc/nxapi.json --files=/var/log/nginx/security.log，无错误提示

• 查看 naxsi log 是否导入成功: curl -XPOST "http://localhost:9200/nxapi/events/_search?pretty" -d '{}'

• 使用 nxtool.py 查看导入 ES 的 naxsi log

root@localhost nxapi]# nxtool.py -c /usr/local/etc/nxapi.json -x
# size :1000
# Whitelist(ing) ratio :
# false 50.0% (total:72/144)
# Top servers :
# 172.16.2.17 77.78% (total:56/72)
# localhost 22.22% (total:16/72)
# Top URI(s) :
# / 77.78% (total:56/72)
# /login.php 22.22% (total:16/72)
# Top Zone(s) :
# ARGS 100.0% (total:72/72)
# Top Peer(s) :
# 10.8.21.60 77.78% (total:56/72)
# 127.0.0.1 22.22% (total:16/72)

• 生成 whitelist: nxtool.py -c nxapi.json -s 172.16.2.17 -f --filter 'uri /' --slack, 错误提示u'Fielddata is disabled on text fields by default. Set fielddata=true on [id] ...

elasticsearch5.exceptions.RequestError: TransportError(400, u'search_phase_execution_exception', u'Fielddata is disabled on text fields by default. Set fielddata=true on [id] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory. Alternatively use a keyword field instead.')

• 解决 fielddata=true问题： 将 text 类型的 fields fielddata置为 true

PUT /nxapi/_mapping/?update_all_types
{
"properties": {
"": {
"type": "text",
"fielddata": true
}
}
}

通过 curl 发送 PUT 请求：

[root@localhost ~]# curl -H "content-type: application/json" -X PUT \
> -d '{ "properties": { "id": { "type": "text", "fielddata": true } } }' \
> "http://localhost:9200/nxapi/_mapping/id?update_all_types"
{"acknowledged":true}

• 重新生成 whitelist:

[root@localhost ~]# nxtool.py -c /usr/local/etc/nxapi.json -s 172.16.2.17 -f --filter 'uri /' --slack
# size :1000
#  template :/usr/local/nxapi/tpl/BODY/precise-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/BODY/site-wide-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/BODY/url-wide-id-BODY-NAME.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/BODY/url-wide-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/BODY/var_name-wide-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/ARGS/precise-id.tpl 
Nb of hits : 28
#  template matched, generating all rules.
5 whitelists ...
#msg: A generic, precise wl tpl (url+var+id)
#Rule (1302) html open tag
#total hits 12
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1302 "mz:$URL:/|$ARGS_VAR:test";
#msg: A generic, precise wl tpl (url+var+id)
#Rule (1003) mysql comment (/*)
#total hits 6
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1003 "mz:$URL:/|$ARGS_VAR:test";
#msg: A generic, precise wl tpl (url+var+id)
#Rule (1003) mysql comment (/*)
#total hits 4
#peers : 10.8.21.60
#uri : /
#var_name : text

BasicRule  wl:1003 "mz:$URL:/|$ARGS_VAR:text";
#msg: A generic, precise wl tpl (url+var+id)
#Rule (1303) html close tag
#total hits 4
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1303 "mz:$URL:/|$ARGS_VAR:test";
#msg: A generic, precise wl tpl (url+var+id)
#Rule (1004) mysql comment (*/)
#total hits 2
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1004 "mz:$URL:/|$ARGS_VAR:test";
#  template :/usr/local/nxapi/tpl/ARGS/site-wide-id.tpl 
Nb of hits : 28
#  template matched, generating all rules.
4 whitelists ...
#msg: A generic, wide (id+zone) wl
#Rule (1302) html open tag
#total hits 12
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1302 "mz:ARGS";
#msg: A generic, wide (id+zone) wl
#Rule (1003) mysql comment (/*)
#total hits 10
#peers : 10.8.21.60
#uri : /
#var_name : test
#var_name : text

BasicRule  wl:1003 "mz:ARGS";
#msg: A generic, wide (id+zone) wl
#Rule (1303) html close tag
#total hits 4
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1303 "mz:ARGS";
#msg: A generic, wide (id+zone) wl
#Rule (1004) mysql comment (*/)
#total hits 2
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1004 "mz:ARGS";
#  template :/usr/local/nxapi/tpl/ARGS/url-wide-id-NAME.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/ARGS/url-wide-id.tpl 
Nb of hits : 28
#  template matched, generating all rules.
4 whitelists ...
#msg: A generic whitelist, true for the whole uri
#Rule (1302) html open tag
#total hits 12
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1302 "mz:$URL:/|ARGS";
#msg: A generic whitelist, true for the whole uri
#Rule (1003) mysql comment (/*)
#total hits 10
#peers : 10.8.21.60
#uri : /
#var_name : test
#var_name : text

BasicRule  wl:1003 "mz:$URL:/|ARGS";
#msg: A generic whitelist, true for the whole uri
#Rule (1303) html close tag
#total hits 4
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1303 "mz:$URL:/|ARGS";
#msg: A generic whitelist, true for the whole uri
#Rule (1004) mysql comment (*/)
#total hits 2
#peers : 10.8.21.60
#uri : /
#var_name : test

BasicRule  wl:1004 "mz:$URL:/|ARGS";
#  template :/usr/local/nxapi/tpl/URI/global-url-0x_in_pircutres.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/URI/site-wide-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/URI/url-wide-id.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/APPS/google_analytics-ARGS.tpl 
Nb of hits : 0
#  template :/usr/local/nxapi/tpl/HEADERS/cookies.tpl 
Nb of hits : 0

安装 Kibana

---

• 从官网下载kibana: wget https://artifacts.elastic.co/downloads/kibana/kibana-5.4.0-linux-x86_64.tar.gz, 注意： Kibana 版本号需要与 Elasticsearch 一致
• 解压缩: tar -vxf kibana-5.4.0-linux-x86_64.tar.gz
• 修改配置文件kibana-5.4.0-linux-x86_64/config/kibana.yml

[root@localhost config]# vi kibana.yml
# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "0.0.0.0"
# The URL of the Elasticsearch instance to use for all your queries.
elasticsearch.url: "http://localhost:9200"

• 运行kibana: [root@localhost config]# ../bin/kibana

• 访问成功
  安装配置成功

• 将 index pattern 配置成 nxapi

• 点击 Discover，如果没有数据，则点击右上角的 time picker，修改 time range
  数据显示