基于elk的springboot web日志收集存储方案
目录
- WEB日志分类处理方案
- ELK安装
-
- Elasticsearch安装
- Logstash安装
- kibana
- 导入依赖
- AOP做接口日志输出
-
- 日志entity类
- AOP Aspect类
- logback-spring.xml配置
- YML配置
WEB日志分类处理方案
web系统的日志按照价值排序分类
-
controller层的接口访问日志（debug日志）
-
自定义包下的其他日志（debug日志）
-
全局日志
① 全局错误日志
② 部分组件的debug日志
③ 部分组件的Info日志
一般来讲，1和2的价值最大，2可基于实际业务情况，进一步划分，如component目录下日志，service层日志等。
ELK安装
Elasticsearch安装
版本：7.6.2
安装前
## 创建数据目录
mkdir -p /mydata/elasticsearch/config
mkdir -p /mydata/elasticsearch/data
mkdir -p /mydata/elasticsearch/plugins
## 限定不限制访问ip
echo "http.host: 0.0.0.0" >> /mydata/elasticsearch/config/elasticsearch.yml
## 修改目录权限
chmod -R 777 /mydata/
部署
docker run --name elasticsearch \
-p 9200:9200 \
-p 9300:9300 \
-e "discovery.type=single-node" \
-e "cluster.name=elasticsearch" \
-e ES_JAVA_OPTS="-Xms64m -Xmx256m" \
-v /mydata/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v /mydata/elasticsearch/data:/usr/share/elasticsearch/data \
-v /mydata/elasticsearch/plugins:/usr/share/elasticsearch/plugins \
-d elasticsearch:7.6.2
# 设置重启
docker update --restart=always elasticsearch
部署后
docker exec -it elasticsearch /bin/bash
## 安装分词插件
#此命令需要在容器中运行
elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip
docker restart elasticsearch
## 防火墙加端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent
firewall-cmd --reload
Logstash安装
版本：7.6.2
部署前
## 创建数据目录
mkdir -p /mydata/logstash
## 修改目录权限
chmod -R 777 /mydata/
# 修改配置文件
vim /mydata/logstash/logstash.conf
修改配置文件logstash.conf
- input：根据不同的日志类型，指定不同的输入端口
- filter：如果是接口日志，去除掉几个属性
- output：指定输出的host和index
input {
tcp {
mode => "server"
host => "0.0.0.0"
port => 4560
codec => json_lines
type => "debug"
}
tcp {
mode => "server"
host => "0.0.0.0"
port => 4561
codec => json_lines
type => "error"
}
tcp {
mode => "server"
host => "0.0.0.0"
port => 4562
codec => json_lines
type => "business"
}
tcp {
mode => "server"
host => "0.0.0.0"
port => 4563
codec => json_lines
type => "record"
}
}
filter{
if [type] == "record" {
mutate {
remove_field => "port"
remove_field => "host"
remove_field => "@version"
}
json {
source => "message"
remove_field => ["message"]
}
}
}
output {
elasticsearch {
hosts => "es:9200"
index => "mall-%{type}-%{+YYYY.MM.dd}"
}
}
部署
docker run --name logstash -p 4560:4560 -p 4561:4561 -p 4562:4562 -p 4563:4563 \
--link elasticsearch:es \
-v /mydata/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf \
-d logstash:7.6.2
# 设置重启
docker update --restart=always logstash
部署后
docker exec -it logstash /bin/bash
# 安装json_lines插件
logstash-plugin install logstash-codec-json_lines
## 防火墙加端口
firewall-cmd --zone=public --add-port=4560/tcp --permanent
firewall-cmd --zone=public --add-port=4561/tcp --permanent
firewall-cmd --zone=public --add-port=4562/tcp --permanent
firewall-cmd --zone=public --add-port=4563/tcp --permanent
firewall-cmd --reload
kibana
版本：7.6.2
部署
docker run --name kibana -p 5601:5601 \
--link elasticsearch:es \
-e "elasticsearch.hosts=http://es:9200" \
-d kibana:7.6.2
# 设置重启
docker update --restart=always kibana
部署后
## 防火墙加端口
firewall-cmd --zone=public --add-port=5601/tcp --permanent
firewall-cmd --reload
导入依赖
<dependencies>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-webartifactId>
dependency>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-starter-aopartifactId>
dependency>
<dependency>
<groupId>net.logstash.logbackgroupId>
<artifactId>logstash-logback-encoderartifactId>
<version>5.3version>
dependency>
<dependency>
<groupId>org.projectlombokgroupId>
<artifactId>lombokartifactId>
<optional>trueoptional>
<scope>providedscope>
<version>${lombok.version}version>
dependency>
dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.bootgroupId>
<artifactId>spring-boot-dependenciesartifactId>
<version>2.7.0version>
<type>pomtype>
<scope>importscope>
dependency>
dependencies>
dependencyManagement>
AOP做接口日志输出
日志entity类
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Controller层的日志封装类
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class WebLog {
/**
* 操作描述
*/
private String description;
/**
* 操作用户
*/
private String username;
/**
* 操作时间
*/
private Long startTime;
/**
* 消耗时间
*/
private Integer spendTime;
/**
* 根路径
*/
private String basePath;
/**
* URI
*/
private String uri;
/**
* URL
*/
private String url;
/**
* 请求类型
*/
private String method;
/**
* IP地址
*/
private String ip;
/**
* 请求参数
*/
private Object parameter;
/**
* 请求返回的结果
*/
private Object result;
}
AOP Aspect类
/**
* 统一日志处理切面
*/
@Slf4j
@Aspect
@Component
@Order(1)
public class WebLogAspect {
@Pointcut("execution(public * com.example.testproject.controller.*.*(..))")
public void webLog() {
}
@Before("webLog()")
public void doBefore(JoinPoint joinPoint) throws Throwable {
}
@AfterReturning(value = "webLog()", returning = "ret")
public void doAfterReturning(Object ret) throws Throwable {
}
@Around("webLog()")
public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable {
long startTime = System.currentTimeMillis();
//获取当前请求对象
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
HttpServletRequest request = attributes.getRequest();
// 记录请求信息
WebLog webLog = new WebLog();
// 运行原始方法
Object result = joinPoint.proceed();
// 拿方法swagger注释信息
Signature signature = joinPoint.getSignature();
MethodSignature methodSignature = (MethodSignature) signature;
Method method = methodSignature.getMethod();
if (method.isAnnotationPresent(ApiOperation.class)) {
ApiOperation apiOperation = method.getAnnotation(ApiOperation.class);
webLog.setDescription(apiOperation.value());
}
// 记录运行时间
long endTime = System.currentTimeMillis();
webLog.setSpendTime((int) (endTime - startTime));
String urlStr = request.getRequestURL().toString(); // url
webLog.setUrl(urlStr); //URL
webLog.setBasePath(StrUtil.removeSuffix(urlStr, URLUtil.url(urlStr).getPath())); //baseurl
webLog.setMethod(request.getMethod()); // GET/POST/PUT/DELETE
webLog.setParameter(getParameter(method, joinPoint.getArgs())); // 方法参数
webLog.setResult(result); // 运行结果
webLog.setStartTime(startTime); //开始时间
webLog.setUri(request.getRequestURI()); // URI
webLog.setIp(request.getRemoteAddr()); // ip
webLog.setUsername(request.getRemoteUser()); // 登录用户名
// 打印日志信息
log.info("{}", JSONUtil.parse(webLog));
// 正常返回方法运行结果
return result;
}
/**
* 根据方法和传入的参数获取请求参数
*/
private Object getParameter(Method method, Object[] args) {
List<Object> argList = new ArrayList<>();
Parameter[] parameters = method.getParameters();
for (int i = 0; i < parameters.length; i++) {
//将RequestBody注解修饰的参数作为请求参数，json参数
RequestBody requestBody = parameters[i].getAnnotation(RequestBody.class);
if (requestBody != null) {
argList.add(args[i]);
}
//将RequestParam注解修饰的参数作为请求参数，其他参数
RequestParam requestParam = parameters[i].getAnnotation(RequestParam.class);
if (requestParam != null) {
Map<String, Object> map = new HashMap<>();
String key = parameters[i].getName();
if (StringUtils.hasLength(requestParam.value())) {
key = requestParam.value();
}
map.put(key, args[i]);
argList.add(map);
}
}
if (argList.size() == 0) {
return null;
} else if (argList.size() == 1) {
return argList.get(0);
} else {
return argList;
}
}
}
logback-spring.xml配置
按照日志分类思路
- 全局日志：全局日志输出到console、debug日志输出到es中mall-debug index中，error日志输出到es中mall-error index中
- 接口日志：由于接口日志通过AOP打印，所以拦截AOP Aspect类的日志信息，输出到es mall-record index中
- 业务日志：其他自定义包下的日志，统一打印到es mall-bussiness index中
另限制部分全局日志模块，只打印info信息
DOCTYPE configuration>
<configuration>
<include resource="org/springframework/boot/logging/logback/defaults.xml"/>
<include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
<springProperty scope="context" name="APP_NAME" source="spring.application.name" defaultValue="springBoot"/>
<springProperty name="LOG_STASH_HOST" scope="context" source="logstash.host" defaultValue="localhost"/>
<property name="PROJECT_NAME" value="mall"/>
<appender name="LOG_STASH_DEBUG" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUGlevel>
filter>
<destination>${LOG_STASH_HOST}:4560destination>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<timestamp>
<timeZone>Asia/ShanghaitimeZone>
timestamp>
<pattern>
<pattern>
{
"level": "%level",
"project": "${PROJECT_NAME:-}",
"service": "${APP_NAME:-}",
"pid": "${PID:-}",
"thread": "%thread",
"class": "%logger",
"message": "%message",
"stack_trace": "%exception{20}"
}
pattern>
pattern>
providers>
encoder>
<connectionStrategy>
<roundRobin>
<connectionTTL>5 minutesconnectionTTL>
roundRobin>
connectionStrategy>
appender>
<appender name="LOG_STASH_ERROR" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>ERRORlevel>
<onMatch>ACCEPTonMatch>
<onMismatch>DENYonMismatch>
filter>
<destination>${LOG_STASH_HOST}:4561destination>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<timestamp>
<timeZone>Asia/ShanghaitimeZone>
timestamp>
<pattern>
<pattern>
{
"level": "%level",
"project": "${PROJECT_NAME:-}",
"service": "${APP_NAME:-}",
"pid": "${PID:-}",
"thread": "%thread",
"class": "%logger",
"message": "%message",
"stack_trace": "%exception{20}"
}
pattern>
pattern>
providers>
encoder>
<connectionStrategy>
<roundRobin>
<connectionTTL>5 minutesconnectionTTL>
roundRobin>
connectionStrategy>
appender>
<appender name="LOG_STASH_RECORD" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>${LOG_STASH_HOST}:4563destination>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<timestamp>
<timeZone>Asia/ShanghaitimeZone>
timestamp>
<pattern>
<pattern>
{
"level": "%level",
"project": "${PROJECT_NAME:-}",
"service": "${APP_NAME:-}",
"pid": "${PID:-}",
"thread": "%thread",
"class": "%logger",
"message": "%message",
"stack_trace": "%exception{20}"
}
pattern>
pattern>
providers>
encoder>
<connectionStrategy>
<roundRobin>
<connectionTTL>5 minutesconnectionTTL>
roundRobin>
connectionStrategy>
appender>
<appender name="LOG_STASH_BUSINESS" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
<destination>${LOG_STASH_HOST}:4562destination>
<encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<timestamp>
<timeZone>Asia/ShanghaitimeZone>
timestamp>
<pattern>
<pattern>
{
"level": "%level",
"project": "${PROJECT_NAME:-}",
"service": "${APP_NAME:-}",
"pid": "${PID:-}",
"thread": "%thread",
"class": "%logger",
"message": "%message",
"stack_trace": "%exception{20}"
}
pattern>
pattern>
providers>
encoder>
<connectionStrategy>
<roundRobin>
<connectionTTL>5 minutesconnectionTTL>
roundRobin>
connectionStrategy>
appender>
<logger name="org.slf4j" level="INFO"/>
<logger name="springfox" level="INFO"/>
<logger name="io.swagger" level="INFO"/>
<logger name="org.springframework" level="INFO"/>
<logger name="org.hibernate.validator" level="INFO"/>
<root level="DEBUG">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="LOG_STASH_DEBUG"/>
<appender-ref ref="LOG_STASH_ERROR"/>
root>
<logger name="com.example.testproject.component.WebLogAspect" level="DEBUG">
<appender-ref ref="LOG_STASH_RECORD"/>
logger>
<logger name="com.example.testproject" level="DEBUG">
<appender-ref ref="LOG_STASH_BUSINESS"/>
logger>
configuration>
YML配置
配置logstash地址信息，同时，要打开全局的debug日志，不然即便在xml中配置了debug日志输出，依然拦截不到debug日志
logstash:
host: 192.168.179.133
logging:
level:
root: debug