http://www.colm.net/files/ragel/ragel-6.10.tar.gz
git clone git://github/01org/hyperscan
GCC, v4.8.1 or higher
Clang, v3.4 or higher (with libstdc++ or libc++)
Intel C++ Compiler v15 or higher
Dependency | Version | Notes |
---|---|---|
CMake | >=2.8.11 | |
Ragel | 6.9 | |
Python | 2.7 | |
Boost | >=1.57 | Boost headers required |
Pcap | >=0.8 | Optional: needed for example code only |
Hyperscan是一款来自于Intel的高性能的正则表达式匹配库。它是基于X86平台以PCRE为原型而开发的,并以BSD许可开源在https://01.org/hyperscan。在支持PCRE的大部分语法的前提下,Hyperscan增加了特定的语法和工作模式来保证其在真实网络场景下的实用性。与此同时,大量高效算法及IntelSIMD*指令的使用实现了Hyperscan的高性能匹配。Hyperscan适用于部署在诸如DPI/IPS/IDS/FW等场景中,目前已经在全球多个客户网络安全方案中得到实际的应用。此外,Hyperscan还支持和开源IDS/IPS产品Snort(https://www.snort.org)和Suricata (https://suricata-ids.org)集成,使其应用更加广泛。
/*
* pcapscan使用并对比了两种匹配模式:BLOCK和STREAM。BLOCK模式时它对单个数据包进行匹配;
* 而STREAM模式下它通过五元组将数据包进行简单分流,并对每条流中的数据进行匹配。STREAM模式
* 可以命中跨越数据包边界的匹配数据(比如,要匹配abc,而a在前一个数据的末尾,而bc在后一个数
* 据包的前端,这两个数据包在一个流中,那么STREAM模式匹配可以命中它,而BLOCK模式不能)。
* Build instructions:
*
* g++ -std=c++11 -O2 -o pcapscan pcapscan.cc $(pkg-config --cflags --libs libhs) -lpcap
*
* Usage:
*
* ./pcapscan [-n repeats]
*
* 规则文件格式为
* ID1: /pcre/
* ID2: /pcre/
*
* 推荐在多核处理器上用taskset隔离处理;
*
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
// We use the BSD primitives throughout as they exist on both BSD and Linux.
#define __FAVOR_BSD
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
using std::cerr;
using std::cout;
using std::endl;
using std::ifstream;
using std::string;
using std::unordered_map;
using std::vector;
// Key for identifying a stream in our pcap input data, using data from its IP
// headers.
struct FiveTuple {
unsigned int protocol;
unsigned int srcAddr;
unsigned int srcPort;
unsigned int dstAddr;
unsigned int dstPort;
// Construct a FiveTuple from a TCP or UDP packet.
FiveTuple(const struct ip *iphdr) {
// IP fields
protocol = iphdr->ip_p;
srcAddr = iphdr->ip_src.s_addr;
dstAddr = iphdr->ip_dst.s_addr;
// UDP/TCP ports
const struct udphdr *uh =
(const struct udphdr *)(((const char *)iphdr) + (iphdr->ip_hl * 4));
srcPort = uh->uh_sport;
dstPort = uh->uh_dport;
}
bool operator==(const FiveTuple &a) const {
return protocol == a.protocol && srcAddr == a.srcAddr &&
srcPort == a.srcPort && dstAddr == a.dstAddr &&
dstPort == a.dstPort;
}
};
// A *very* simple hash function, used when we create an unordered_map of
// FiveTuple objects.
struct FiveTupleHash {
size_t operator()(const FiveTuple &x) const {
return x.srcAddr ^ x.dstAddr ^ x.protocol ^ x.srcPort ^ x.dstPort;
}
};
// Helper function. See end of file.
static bool payloadOffset(const unsigned char *pkt_data, unsigned int *offset,
unsigned int *length);
// Match event handler: called every time Hyperscan finds a match.
//匹配命中回调函数
static
int onMatch(unsigned int id, unsigned long long from, unsigned long long to,
unsigned int flags, void *ctx) {
// Our context points to a size_t storing the match count
size_t *matches = (size_t *)ctx;
(*matches)++;
return 0; // continue matching
}
// Simple timing class,简单的定时器类
class Clock {
public:
void start() {
time_start = std::chrono::system_clock::now();
}
void stop() {
time_end = std::chrono::system_clock::now();
}
double seconds() const {
std::chrono::duration delta = time_end - time_start;
return delta.count();
}
private:
std::chrono::time_point time_start, time_end;
};
// Class wrapping all state associated with the benchmark
class Benchmark {
private:
// Packet data to be scanned.
vector packets;
// The stream ID to which each packet belongs
vector stream_ids;
// Map used to construct stream_ids
unordered_map stream_map;
// Hyperscan compiled database (streaming mode)
const hs_database_t *db_streaming;
// Hyperscan compiled database (block mode)
const hs_database_t *db_block;
// Hyperscan temporary scratch space (used in both modes)
hs_scratch_t *scratch;
// Vector of Hyperscan stream state (used in streaming mode)
vector streams;
// Count of matches found during scanning
size_t matchCount;
public:
Benchmark(const hs_database_t *streaming, const hs_database_t *block)
: db_streaming(streaming), db_block(block), scratch(nullptr),
matchCount(0) {
// Allocate enough scratch space to handle either streaming or block
// mode, so we only need the one scratch region.
//Benchmark构造函数中,为接下来的匹配分配足够的临时数据空间(scratch space)。这里有一个技巧:
//1)BLOCK和STREAM模式的匹配只需共用一个scratch;
//2)这个scratch足够大,方法是调用两次,在第2次调用时hyperscan如果发现空间不够会进行增加。
hs_error_t err = hs_alloc_scratch(db_streaming, &scratch);
if (err != HS_SUCCESS) {
cerr << "ERROR: could not allocate scratch space. Exiting." << endl;
exit(-1);
}
// This second call will increase the scratch size if more is required
// for block mode.
err = hs_alloc_scratch(db_block, &scratch);
if (err != HS_SUCCESS) {
cerr << "ERROR: could not allocate scratch space. Exiting." << endl;
exit(-1);
}
}
~Benchmark() {
// Free scratch region
hs_free_scratch(scratch);
}
// Read a set of streams from a pcap file
//此流的建立知识做简单按包中的顺序建立,没有考虑丢包、重传、乱序的情况;
bool readStreams(const char *pcapFile) {
// Open PCAP file for input
char errbuf[PCAP_ERRBUF_SIZE];
pcap_t *pcapHandle = pcap_open_offline(pcapFile, errbuf);
if (pcapHandle == nullptr) {
cerr << "ERROR: Unable to open pcap file \"" << pcapFile
<< "\": " << errbuf << endl;
return false;
}
struct pcap_pkthdr pktHeader;
const unsigned char *pktData;
while ((pktData = pcap_next(pcapHandle, &pktHeader)) != nullptr) {
unsigned int offset = 0, length = 0;
if (!payloadOffset(pktData, &offset, &length)) {
continue;
}
// Valid TCP or UDP packet
const struct ip *iphdr = (const struct ip *)(pktData
+ sizeof(struct ether_header));
const char *payload = (const char *)pktData + offset;
//5元组进行流hash
size_t id = stream_map.insert(std::make_pair(FiveTuple(iphdr),
stream_map.size())).first->second;
packets.push_back(string(payload, length));
stream_ids.push_back(id); //用向量作为hash表;
}
pcap_close(pcapHandle);
return !packets.empty();
}
// Return the number of bytes scanned
size_t bytes() const {
size_t sum = 0;
for (const auto &packet : packets) {
sum += packet.size();
}
return sum;
}
// Return the number of matches found.
size_t matches() const {
return matchCount;
}
// Clear the number of matches found.
void clearMatches() {
matchCount = 0;
}
// Open a Hyperscan stream for each stream in stream_ids
// 打开一条流
void openStreams() {
streams.resize(stream_map.size());
for (auto &stream : streams) {
hs_error_t err = hs_open_stream(db_streaming, 0, &stream);
if (err != HS_SUCCESS) {
cerr << "ERROR: Unable to open stream. Exiting." << endl;
exit(-1);
}
}
}
// Close all open Hyperscan streams (potentially generating any
// end-anchored matches)
void closeStreams() {
for (auto &stream : streams) {
hs_error_t err = hs_close_stream(stream, scratch, onMatch,
&matchCount);
if (err != HS_SUCCESS) {
cerr << "ERROR: Unable to close stream. Exiting." << endl;
exit(-1);
}
}
}
// Scan each packet (in the ordering given in the PCAP file) through
// Hyperscan using the streaming interface.
// 扫描所有的流,进行匹配
void scanStreams() {
//如何利用到时间是关键
for (size_t i = 0; i != packets.size(); ++i) {
const std::string &pkt = packets[i];
/* 函数原型
* hs_error_t hs_scan_stream(hs_stream_t * id,
const char * data,
unsigned int length,
unsigned int flags,
hs_scratch_t * scratch,
match_event_handler onEvent,
void * ctxt)
* */
hs_error_t err = hs_scan_stream(streams[stream_ids[i]],
pkt.c_str(), pkt.length(), 0,
scratch, onMatch, &matchCount);
if (err != HS_SUCCESS) {
cerr << "ERROR: Unable to scan packet. Exiting." << endl;
exit(-1);
}
}
}
// Scan each packet (in the ordering given in the PCAP file) through
// Hyperscan using the block-mode interface.
void scanBlock() {
for (size_t i = 0; i != packets.size(); ++i) {
const std::string &pkt = packets[i];
hs_error_t err = hs_scan(db_block, pkt.c_str(), pkt.length(), 0,
scratch, onMatch, &matchCount);
if (err != HS_SUCCESS) {
cerr << "ERROR: Unable to scan packet. Exiting." << endl;
exit(-1);
}
}
}
// Display some information about the compiled database and scanned data.
void displayStats() {
size_t numPackets = packets.size();
size_t numStreams = stream_map.size();
size_t numBytes = bytes();
hs_error_t err;
cout << numPackets << " packets in " << numStreams
<< " streams, totalling " << numBytes << " bytes." << endl;
cout << "Average packet length: " << numBytes / numPackets << " bytes."
<< endl;
cout << "Average stream length: " << numBytes / numStreams << " bytes."
<< endl;
cout << endl;
size_t dbStream_size = 0;
err = hs_database_size(db_streaming, &dbStream_size);
if (err == HS_SUCCESS) {
cout << "Streaming mode Hyperscan database size : "
<< dbStream_size << " bytes." << endl;
} else {
cout << "Error getting streaming mode Hyperscan database size"
<< endl;
}
size_t dbBlock_size = 0;
err = hs_database_size(db_block, &dbBlock_size);
if (err == HS_SUCCESS) {
cout << "Block mode Hyperscan database size : "
<< dbBlock_size << " bytes." << endl;
} else {
cout << "Error getting block mode Hyperscan database size"
<< endl;
}
size_t stream_size = 0;
err = hs_stream_size(db_streaming, &stream_size);
if (err == HS_SUCCESS) {
cout << "Streaming mode Hyperscan stream state size: "
<< stream_size << " bytes (per stream)." << endl;
} else {
cout << "Error getting stream state size" << endl;
}
}
};
// helper function - see end of file
static void parseFile(const char *filename, vector &patterns,
vector &flags, vector &ids);
//mode 模式为块模式和流模式
static hs_database_t *buildDatabase(const vector &expressions,
const vector flags,
const vector ids,
unsigned int mode) {
hs_database_t *db;
hs_compile_error_t *compileErr;
hs_error_t err;
Clock clock;
clock.start();
//hs_compile_multi的调用,此函数用来编译多个正则表达式,从代码可见除了mode参数,BLOCK和STREAM模式都使用这一API。
err = hs_compile_multi(expressions.data(), flags.data(), ids.data(),
expressions.size(), mode, nullptr, &db, &compileErr);
clock.stop();
if (err != HS_SUCCESS) {
if (compileErr->expression < 0) {
// The error does not refer to a particular expression.
cerr << "ERROR: " << compileErr->message << endl;
} else {
cerr << "ERROR: Pattern '" << expressions[compileErr->expression]
<< "' failed compilation with error: " << compileErr->message
<< endl;
}
// As the compileErr pointer points to dynamically allocated memory, if
// we get an error, we must be sure to release it. This is not
// necessary when no error is detected.
hs_free_compile_error(compileErr);
exit(-1);
}
cout << "Hyperscan " << (mode == HS_MODE_STREAM ? "streaming" : "block")
<< " mode database compiled in " << clock.seconds() << " seconds."
<< endl;
return db;
}
/**
* This function will read in the file with the specified name, with an
* expression per line, ignoring lines starting with '#' and build a Hyperscan
* database for it.
* 这个函数将读一个指定文件名文件,提取规则表达式,忽略#注释。
*/
static void databasesFromFile(const char *filename,
hs_database_t **db_streaming,
hs_database_t **db_block) {
// hs_compile_multi requires three parallel arrays containing the patterns,
// hs_compile_multi 需要三个平行数组容器存储模式,标识和ID。
// flags and ids that we want to work with. To achieve this we use
// vectors and new entries onto each for each valid line of input from
// the pattern file.
//用三个向量来存储相关条目;
vector patterns; //模式
vector flags; //标识
vector ids; //规则ID
// do the actual file reading and string handling
parseFile(filename, patterns, flags, ids);
// Turn our vector of strings into a vector of char*'s to pass in to
// hs_compile_multi. (This is just using the vector of strings as dynamic
// storage.)
//将vector中字符串对象转换成一个C字符串指针传入到hs_compile_multi;
/*函数原型: hs_error_t hs_compile_multi(const char *const * expressions,
const unsigned int * flags,
const unsigned int * ids,
unsigned int elements,
unsigned int mode,
const hs_platform_info_t * platform,
hs_database_t ** db,
hs_compile_error_t ** error)*/
vector cstrPatterns;
for (const auto &pattern : patterns) {
cstrPatterns.push_back(pattern.c_str());
}
cout << "Compiling Hyperscan databases with " << patterns.size()
<< " patterns." << endl;
/* 构建流和块数据库 */
*db_streaming = buildDatabase(cstrPatterns, flags, ids, HS_MODE_STREAM);
*db_block = buildDatabase(cstrPatterns, flags, ids, HS_MODE_BLOCK);
}
static void usage(const char *prog) {
cerr << "Usage: " << prog << " [-n repeats] " << endl;
}
// Main entry point.
int main(int argc, char **argv) {
unsigned int repeatCount = 1;
// Process command line arguments.
int opt;
while ((opt = getopt(argc, argv, "n:")) != -1) {
switch (opt) {
case 'n':
repeatCount = atoi(optarg);
break;
default:
usage(argv[0]);
exit(-1);
}
}
if (argc - optind != 2) {
usage(argv[0]);
exit(-1);
}
const char *patternFile = argv[optind];
const char *pcapFile = argv[optind + 1];
// Read our pattern set in and build Hyperscan databases from it.
cout << "Pattern file: " << patternFile << endl;
/* 构建规则数据库 ,分为流模式和块模式 */
hs_database_t *db_streaming, *db_block;
databasesFromFile(patternFile, &db_streaming, &db_block);
// Read our input PCAP file in
Benchmark bench(db_streaming, db_block); //构建基准测试类
cout << "PCAP input file: " << pcapFile << endl;
if (!bench.readStreams(pcapFile)) {
cerr << "Unable to read packets from PCAP file. Exiting." << endl;
exit(-1);
}
if (repeatCount != 1) {
cout << "Repeating PCAP scan " << repeatCount << " times." << endl;
}
bench.displayStats();
Clock clock;
// Streaming mode scans.
double secsStreamingScan = 0.0, secsStreamingOpenClose = 0.0;
for (unsigned int i = 0; i < repeatCount; i++) {
// Open streams.
clock.start();
bench.openStreams();
clock.stop();
secsStreamingOpenClose += clock.seconds();
// Scan all our packets in streaming mode.
clock.start();
bench.scanStreams();
clock.stop();
secsStreamingScan += clock.seconds();
// Close streams.
clock.start();
bench.closeStreams();
clock.stop();
secsStreamingOpenClose += clock.seconds();
}
// Collect data from streaming mode scans.
size_t bytes = bench.bytes();
double tputStreamScanning = (bytes * 8 * repeatCount) / secsStreamingScan;
double tputStreamOverhead = (bytes * 8 * repeatCount) / (secsStreamingScan + secsStreamingOpenClose);
size_t matchesStream = bench.matches();
double matchRateStream = matchesStream / ((bytes * repeatCount) / 1024.0); // matches per kilobyte
// Scan all our packets in block mode.
bench.clearMatches();
clock.start();
for (unsigned int i = 0; i < repeatCount; i++) {
bench.scanBlock();
}
clock.stop();
double secsScanBlock = clock.seconds();
// Collect data from block mode scans.
double tputBlockScanning = (bytes * 8 * repeatCount) / secsScanBlock;
size_t matchesBlock = bench.matches();
double matchRateBlock = matchesBlock / ((bytes * repeatCount) / 1024.0); // matches per kilobyte
cout << endl << "Streaming mode:" << endl << endl;
cout << " Total matches: " << matchesStream << endl;
cout << std::fixed << std::setprecision(4);
cout << " Match rate: " << matchRateStream << " matches/kilobyte" << endl;
cout << std::fixed << std::setprecision(2);
cout << " Throughput (with stream overhead): "
<< tputStreamOverhead/1000000 << " megabits/sec" << endl;
cout << " Throughput (no stream overhead): "
<< tputStreamScanning/1000000 << " megabits/sec" << endl;
cout << endl << "Block mode:" << endl << endl;
cout << " Total matches: " << matchesBlock << endl;
cout << std::fixed << std::setprecision(4);
cout << " Match rate: " << matchRateBlock << " matches/kilobyte" << endl;
cout << std::fixed << std::setprecision(2);
cout << " Throughput: "
<< tputBlockScanning/1000000 << " megabits/sec" << endl;
cout << endl;
if (bytes < (2*1024*1024)) {
cout << endl << "WARNING: Input PCAP file is less than 2MB in size." << endl
<< "This test may have been too short to calculate accurate results." << endl;
}
// Close Hyperscan databases
hs_free_database(db_streaming);
hs_free_database(db_block);
return 0;
}
/**
* Helper function to locate the offset of the first byte of the payload in the
* given ethernet frame. Offset into the packet, and the length of the payload
* are returned in the arguments @a offset and @a length.
* 主要是pcap格式有一个头自己的标识头, 标准的协议层次eth-ip-tcp/udp,没有特殊情况下的;
*/
static bool payloadOffset(const unsigned char *pkt_data, unsigned int *offset,
unsigned int *length) {
const ip *iph = (const ip *)(pkt_data + sizeof(ether_header));
const tcphdr *th = nullptr;
// Ignore packets that aren't IPv4
// 忽略不是IPV4的数据包
if (iph->ip_v != 4) {
return false;
}
// Ignore fragmented packets.
// 忽略IP分片包,根据标识味
if (iph->ip_off & htons(IP_MF|IP_OFFMASK)) {
return false;
}
// IP header length, and transport header length.
unsigned int ihlen = iph->ip_hl * 4;
unsigned int thlen = 0;
switch (iph->ip_p) {
case IPPROTO_TCP:
th = (const tcphdr *)((const char *)iph + ihlen);
thlen = th->th_off * 4;
break;
case IPPROTO_UDP:
thlen = sizeof(udphdr);
break;
default:
return false;
}
*offset = sizeof(ether_header) + ihlen + thlen;
*length = sizeof(ether_header) + ntohs(iph->ip_len) - *offset;
return *length != 0;
}
//正则模式标识
static unsigned parseFlags(const string &flagsStr) {
unsigned flags = 0;
for (const auto &c : flagsStr) {
switch (c) {
case 'i':
flags |= HS_FLAG_CASELESS; break;
case 'm':
flags |= HS_FLAG_MULTILINE; break; //如果regexp里出现了^或者$, 那么by default只会匹配第一行. 设置了Multiline,会匹配所有行.
case 's':
flags |= HS_FLAG_DOTALL; break; //默认情况下, .不会匹配换行符, 设置了Dotall模式, .会匹配所有字符包括换行符
case 'H':
flags |= HS_FLAG_SINGLEMATCH; break;
case 'V':
flags |= HS_FLAG_ALLOWEMPTY; break;
case '8':
flags |= HS_FLAG_UTF8; break;
case 'W':
flags |= HS_FLAG_UCP; break;
case '\r': // stray carriage-return
break;
default:
cerr << "Unsupported flag \'" << c << "\'" << endl;
exit(-1);
}
}
return flags;
}
static void parseFile(const char *filename, vector &patterns,
vector &flags, vector &ids) {
ifstream inFile(filename);
if (!inFile.good()) {
cerr << "ERROR: Can't open pattern file \"" << filename << "\"" << endl;
exit(-1);
}
for (unsigned i = 1; !inFile.eof(); ++i) {
string line;
getline(inFile, line);
// if line is empty, or a comment, we can skip it
if (line.empty() || line[0] == '#') {
continue;
}
// otherwise, it should be ID:PCRE, e.g.
// 10001:/foobar/is
size_t colonIdx = line.find_first_of(':');
if (colonIdx == string::npos) {
cerr << "ERROR: Could not parse line " << i << endl;
exit(-1);
}
// we should have an unsigned int as an ID, before the colon
unsigned id = std::stoi(line.substr(0, colonIdx).c_str());
// rest of the expression is the PCRE
const string expr(line.substr(colonIdx + 1));
size_t flagsStart = expr.find_last_of('/');
if (flagsStart == string::npos) {
cerr << "ERROR: no trailing '/' char" << endl;
exit(-1);
}
string pcre(expr.substr(1, flagsStart - 1));
string flagsStr(expr.substr(flagsStart + 1, expr.size() - flagsStart));
unsigned flag = parseFlags(flagsStr);
patterns.push_back(pcre);
flags.push_back(flag);
ids.push_back(id);
}
}
包: 3.2M包 这个特征跨两个包
结果:
root@dev-desktop:/home/pangyemeng/hyperscan-master/build/bin# ./pcapscan test get.pcap
Pattern file: test
Compiling Hyperscan databases with 1 patterns.
Hyperscan streaming mode database compiled in 0.00425785 seconds.
Hyperscan block mode database compiled in 0.00526345 seconds.
PCAP input file: get.pcap
2 packets in 1 streams, totalling 2384 bytes.
Average packet length: 1192 bytes.
Average stream length: 2384 bytes.
Streaming mode Hyperscan database size : 4984 bytes.
Block mode Hyperscan database size : 5048 bytes.
Streaming mode Hyperscan stream state size: 51 bytes (per stream).
Streaming mode:
Total matches: 1
Match rate: 0.4295 matches/kilobyte
Throughput (with stream overhead): 225.22 megabits/sec
Throughput (no stream overhead): 248.09 megabits/sec
Block mode:
Total matches: 0
Match rate: 0.0000 matches/kilobyte
Throughput: 2129.76 megabits/sec
WARNING: Input PCAP file is less than 2MB in size.
This test may have been too short to calculate accurate results.
0x05 吸引我的地方
1、跨包检测;
2、流检测;
如果能做出良好的设计,我想做DPI还是不错的选择;