一、YAF的安装
环境：ubuntu16和debian9.5通过
（1）下载最新版本的YAF
https://tools.netsa.cert.org/yaf/download.html
（2）编译YAF
1、首先需要安装glib的相关库，libfixbuf需要
https://unix.stackexchange.com/questions/182156/gentoo-no-package-glib-2-0-found
sudo apt-get install libperl-dev
sudo apt-get install libgtk2.0-dev
2、安装libfixbuf
https://tools.netsa.cert.org/fixbuf/download.html
$ tar -xvzf libfixbuf-2.1.0.tar.gz
$ cd libfixbuf-2.1.0
$ ./configure
$ make
$ make install
3、安装YAF
如要使用ndpi，从下面链接下载文件并编译
https://sourceforge.net/projects/ntop/files/nDPI/old/nDPI-1.7.tar.gz/download
$ tar -xvzf yaf-2.6.0.tar.gz
$ cd yaf-2.6.0
$ ./configure --enable-applabel --enable-plugins --enable-ndpi
$ make
$sudo make install
ndpi目前只支持ndpi1.7版本
4、最后运行：sudo ldconfig
参考：https://tools.netsa.cert.org/yaf/libyaf/yaf_orcus.html
二、安装kafka
参考：https://www.cnblogs.com/runnerjack/p/8592036.html
三、logstash的配置文件
1、针对kafka的YAF流量采集
input {
kafka {
topics => ["yaf"]
group_id => "yaf_http"
bootstrap_servers => "localhost:9092"
auto_offset_reset => "earliest"
type => "yaf"
}
}
filter{
if [type] == "yaf" {
csv{
columns => ["starttime","endtime","duration","rtt","proto","sip","sport","dip","dport","iflags","uflags","riflags","ruflags","isn","risn","tag","rtag","pkt","oct","rpkt","roct","applabel","endreason"]
separator => "|"
}
mutate {
gsub => ["duration"," ",""]
}
mutate {
gsub => ["rtt"," ",""]
}
mutate {
gsub => ["proto"," ",""]
}
mutate {
gsub => ["sip"," ",""]
}
mutate {
gsub => ["sport"," ",""]
}
mutate {
gsub => ["dip"," ",""]
}
mutate {
gsub => ["dport"," ",""]
}
mutate {
gsub => ["iflags"," ",""]
}
mutate {
gsub => ["uflags"," ",""]
}
mutate {
gsub => ["riflags"," ",""]
}
mutate {
gsub => ["ruflags"," ",""]
}
mutate {
gsub => ["isn"," ",""]
}
mutate {
gsub => ["risn"," ",""]
}
mutate {
gsub => ["tag"," ",""]
}
mutate {
gsub => ["rtag"," ",""]
}
mutate {
gsub => ["pkt"," ",""]
}
mutate {
gsub => ["oct"," ",""]
}
mutate {
gsub => ["rpkt"," ",""]
}
mutate {
gsub => ["roct"," ",""]
}
mutate {
gsub => ["applabel"," ",""]
}
mutate {
gsub => ["endreason"," ",""]
}
}
date {
match => ["starttime","yyyy-MM-dd HH:mm:ss.SSS"]
}
}
output {
if [type] == "yaf" {
stdout { codec => "rubydebug" }
elasticsearch{
hosts => ["localhost:9200"]
index => "logstash-yaf1"
document_type => "yafdoc"
}
}
}
2、针对日志文件的YAF流量采集
input {
file {
type => "yaf1"
start_position => "end"
sincedb_path => "$HOME/logstash-6.2.2/bin/log/.bro_conn_sincedb"
#Edit the following path to reflect the location of your log files. You can also change the extension if you use something else
path => "$HOME/test.txt"
}
}
filter{
if [type] == "yaf1" {
csv{
columns => ["starttime","endtime","duration","rtt","proto","sip","sport","dip","dport","iflags","uflags","riflags","ruflags","isn","risn","tag","rtag","pkt","oct","rpkt","roct","applabel","endreason"]
separator => "|"
}
mutate {
gsub => ["duration"," ",""]
}
mutate {
gsub => ["rtt"," ",""]
}
mutate {
gsub => ["proto"," ",""]
}
mutate {
gsub => ["sip"," ",""]
}
mutate {
gsub => ["sport"," ",""]
}
mutate {
gsub => ["dip"," ",""]
}
mutate {
gsub => ["dport"," ",""]
}
mutate {
gsub => ["iflags"," ",""]
}
mutate {
gsub => ["uflags"," ",""]
}
mutate {
gsub => ["riflags"," ",""]
}
mutate {
gsub => ["ruflags"," ",""]
}
mutate {
gsub => ["isn"," ",""]
}
mutate {
gsub => ["risn"," ",""]
}
mutate {
gsub => ["tag"," ",""]
}
mutate {
gsub => ["rtag"," ",""]
}
mutate {
gsub => ["pkt"," ",""]
}
mutate {
gsub => ["oct"," ",""]
}
mutate {
gsub => ["rpkt"," ",""]
}
mutate {
gsub => ["roct"," ",""]
}
mutate {
gsub => ["applabel"," ",""]
}
mutate {
gsub => ["endreason"," ",""]
}
}
date {
match => ["starttime","yyyy-MM-dd HH:mm:ss.SSS"]
}
}
output {
if [type] == "yaf1" {
stdout { codec => "rubydebug" }
elasticsearch{
hosts => ["localhost:9200"]
index => "logstash-yaffile"
document_type => "yafdoc"
}
}
}
四、运行yaf程序
1、针对kafak的流量采集程序
sudo yaf --in ens33 --live pcap--applabel --max-payload=383| yafscii --tabular | $HOME/kafka_2.12-1.0.0/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic yaf
2、针对日志文件的流量采集程序
sudo yaf --in ens33 --live pcap --applabel --max-payload=383 | yafscii --tabular --out test.txt