Filebeat log @timestamp处理
环境：
Elasticsearch版本：5.6.9
Filebeat版本：6.3.1（为了获取ip部分信息，而6.3.1的filebeat中还没有该功能，实际用的是master分支编译的）日志格式：
%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> 样例：
2018-06-05 10:18:36.576 [DubboServerHandler-10.138.86.239:20801-thread-998] DEBUG c.h.H.c.d.s.p.d.b.C.queryByCd_COUNT - <== Total: 1
2018-06-05 10:18:36.660 [DubboServerHandler-10.138.86.239:20801-thread-998] DEBUG c.h.H.c.d.s.p.d.b.C.queryByCdMMM_COUNT - ==> Preparing: SELECT count(0) FROM (SELECT RTRIM(CCMM.ROW_ID) AS ROW_ID, CCMM.LANGUAGE, RTRIM(CCMM.PARENT_ID) AS PARENT_ID, CCMM.MMM_TYPE, CCMM.MMM_C, CCMM.SAP_C, CCMM.TAX_C, CCMM.MMM_DESC, CCMM.MMM_UNIT, CCMM.ORDER_STANDARD, CCMM.SEND_STANDARD, CCMM.PROVIDER_C, CCMM.DEPARTMENT_PP, CD.DEPARTMENT_DESC, CD.DEPARTMENT_C, CCMM.PRODTYPE_C, CCMM.PRODUCT_ID, CCMM.COST_P, CCMM.SETTLEMENT_P, CCMM.RETAIL_P, CCMM.SCRAP_PS, CCMM.DULL_PS, CCMM.MMM_WEIGHT, CCMM.MMM_SIZE, CCMM.MMM_SPECIFICATIONS, CCMM.MMM_COLOR, CCMM.PROVIDE_CYCLE, CCMM.SHELF_LIFE, CCMM.PRODUCT_S, CCMM.DELIVERY_S, CCMM.EXPEND_PROPERTY, CCMM.PHYSICS_PROPERTY, CCMM.BACK_FLAG, CCMM.REPAIR_FLAG, CCMM.REPLACE_FLAG, CCMM.KEY_FLAG, CCMM.HIGH_VALUE_FLAG, CCMM.SEMI_FINISHED_FLAG, CCMM.BK_FLAG, CCMM.INDEPENDENT_PACKING, CCMM.REPAIR_RATE, CCMM.A_PLAN_FLAG, CCMM.PO_TYPE, CCMM.INPUT_SAP_FLAG, CCMM.USING_FLAG, CCMM.CLASSIFY_C, CCMM.IMPORT_FLAG, CCMM.VC_FLAG, CCMM.VC_DATE, CCMM.SHARED_FLAG, CCMM.REMARK, CCMM.ARCHIVE_BASE_DATE, CCMM.CREATED_BY, CCMM.CREATED_DATE, CCMM.LAST_UPDATED_BY, CCMM.LAST_UPDATE_DATE, CCMM.RECORD_VERSION, CCMM.DELETED_FLAG, CCMM.DELETED_BY, CCMM.DELETION_DATE, CCMM.ATTRIBUTE1, CCMM.ATTRIBUTE2, CCMM.ATTRIBUTE3, CCMM.ATTRIBUTE4, CCMM.ATTRIBUTE5, CCMM.ATTRIBUTE6, CCMM.ATTRIBUTE7, CCMM.ATTRIBUTE8, CCMM.ATTRIBUTE9, CCMM.ATTRIBUTE10, CCMM.ATTRIBUTE12, CCMM.ATTRIBUTE11, CCMM.BIZ_ORG_C, CCMM.V_NO, CCMM.MMM_SN, CCMM.VOLUME_NO, CCMM.WARRANTY_P, CCMM.MMM_ETHNIC_GROUP, CCMM.LOCATION_TYPE, CCMM.CHARGE_FLAG, CCMM.PACKAGE_P, CCMM.FACTORY_C, CCMM.FACTORY_NAME, CCMM.prodtype_Name, CCMM.FACTORY_ID, CCMM.Sale_FLAG, CCMM.TY_FLAG FROM HHHHHHHHH_SP.dboooo.CD_MMM CCMM LEFT JOIN HHHHHHHHH_SP.dboooo.CD_DEPARTMENT CD ON CCMM.DEPARTMENT_PP = CD.PARENT_ID WHERE CCMM.MMM_C = ? AND CCMM.DELETED_FLAG = ?) table_count
2018-06-05 10:18:36.661 [DubboServerHandler-10.138.86.239:20801-thread-998] DEBUG c.h.H.c.d.s.p.d.b.C.queryByCdMMM_COUNT - ==> Parameters: 0020507744(String), N(String)使用Elasticsearch Ingest Node
编写pipeline如下：
{
"timestamp-pipeline-id": {
"description": "timestamp pipeline",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{TIMESTAMP_ISO8601:timestamp} "
]
},
"remove": {
"field": "@timestamp"
}
},
{
"date": {
"field": "timestamp",
"formats": [
"yyyy-MM-dd HH:mm:ss.SSS"
]
},
"remove": {
"field": "timestamp"
}
}
],
"on_failure": [
{
"set": {
"field": "_index",
"value": "failed-{{ _index }}"
}
}
]
}
}在filebeat output中进行如下配置：
output.elasticsearch:
hosts: ["10.158.75.294:9200"]
pipeline: "timestamp-pipeline-id"处理后的样例：
{
"_index": "filebeat-7.0.0-alpha1-2018.07.18",
"_type": "doc",
"_id": "AWSsHlkCR0KAk4F5NPlL",
"_score": 1.7230201,
"_source": {
"offset": 328,
"prospector": {
"type": "log"
},
"source": "/usr/local/data/logs/jiankunking/sp-barcode-2018-06-19 18.0.log",
"message": "2018-06-19 18:00:00.006 [DubboServerHandler-10.138.334.78:20809-thread-97] DEBUG com.jiankunking.barcode.dao.SeqDao.selectSeq - <== Total: 1",
"input": {
"type": "log"
},
"@timestamp": "2018-06-19T18:00:00.006Z",
"beat": {
"hostname": "jiankunking-123-6",
"name": "jiankunking-123-6",
"version": "7.0.0-alpha1"
},
"host": {
"os": {
"codename": "Core",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": true,
"ip": ["10.138.334.78",
"fe80::250:56ff:fe9e:f23a",
"192.168.122.1",
"172.17.0.1",
"fe80::42:efff:fefa:f021",
"fe80::683b:95ff:fe7f:195a",
"fe80::3031:abff:fe3f:1f9a",
"fe80::dce4:22ff:fef5:2487",
"fe80::7897:b7ff:febf:1160",
"fe80::8006:d1ff:fe51:7834",
"fe80::344d:75ff:feb0:3cd5",
"fe80::70b8:40ff:fe02:78de"],
"name": "jiankunking-123-6",
"id": "edcbe58e37b844db91a6a41667323d9d",
"mac": ["00:50:56:9e:f2:3a",
"52:54:00:a5:d6:98",
"52:54:00:a5:d6:98",
"02:42:ef:fa:f0:21",
"6a:3b:95:7f:19:5a",
"32:31:ab:3f:1f:9a",
"de:e4:22:f5:24:87",
"7a:97:b7:bf:11:60",
"82:06:d1:51:78:34",
"36:4d:75:b0:3c:d5",
"72:b8:40:02:78:de"],
"architecture": "x86_64"
},
"fields": {
"project": "jiankunking",
"type": "log"
}
}
},
{
"_index": "filebeat-7.0.0-alpha1-2018.07.18",
"_type": "doc",
"_id": "AWSsHlm3R0KAk4F5NPlS",
"_score": 1.7230201,
"_source": {
"offset": 695916,
"prospector": {
"type": "log"
},
"source": "/usr/local/data/logs/jiankunking/jiankunking-2018-07-16.5.log",
"message": "2018-07-16 14:78:34.649 [New I/O client worker #1-3] ERROR com.alibaba.dubbo.remoting.transport.AbstractCodec - Data length too large: 1314982449, max payload: 8388608, channel: NettyChannel [channel=[id: 0x575e572f, /172.17.0.5:39897 => /10.138.334.78:20804]]\njava.io.IOException: Data length too large: 1314982449, max payload: 8388608, channel: NettyChannel [channel=[id: 0x575e572f, /172.17.0.5:39897 => /10.138.334.78:20804]]\n\tat com.alibaba.dubbo.remoting.transport.AbstractCodec.checkPayload(AbstractCodec.java:49)\n\tat com.alibaba.dubbo.remoting.exchange.codec.ExchangeCodec.decode(ExchangeCodec.java:116)\n\tat com.alibaba.dubbo.remoting.exchange.codec.ExchangeCodec.decode(ExchangeCodec.java:87)\n\tat com.alibaba.dubbo.rpc.protocol.dubbo.DubboCountCodec.decode(DubboCountCodec.java:47)\n\tat com.alibaba.dubbo.remoting.transport.netty.NettyCodecAdapter$InternalDecoder.messageReceived(NettyCodecAdapter.java:134)\n\tat org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:80)\n\tat org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)\n\tat org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:789)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:274)\n\tat org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:261)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:349)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.processSelectedKeys(NioWorker.java:280)\n\tat org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:200)\n\tat org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)\n\tat org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:44)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)",
"input": {
"type": "log"
},
"@timestamp": "2018-07-16T14:78:34.649Z",
"beat": {
"hostname": "jiankunking-123-6",
"name": "jiankunking-123-6",
"version": "7.0.0-alpha1"
},
"host": {
"os": {
"codename": "Core",
"family": "redhat",
"version": "7 (Core)",
"platform": "centos"
},
"containerized": true,
"ip": ["10.138.334.78",
"fe80::250:56ff:fe9e:f23a",
"192.168.122.1",
"172.17.0.1",
"fe80::42:efff:fefa:f021",
"fe80::683b:95ff:fe7f:195a",
"fe80::3031:abff:fe3f:1f9a",
"fe80::dce4:22ff:fef5:2487",
"fe80::7897:b7ff:febf:1160",
"fe80::8006:d1ff:fe51:7834",
"fe80::344d:75ff:feb0:3cd5",
"fe80::70b8:40ff:fe02:78de"],
"name": "jiankunking-123-6",
"id": "edcbe58e37b844db91a6a41667323d9d",
"mac": ["00:50:56:9e:f2:3a",
"52:54:00:a5:d6:98",
"52:54:00:a5:d6:98",
"02:42:ef:fa:f0:21",
"6a:3b:95:7f:19:5a",
"32:31:ab:3f:1f:9a",
"de:e4:22:f5:24:87",
"7a:97:b7:bf:11:60",
"82:06:d1:51:78:34",
"36:4d:75:b0:3c:d5",
"72:b8:40:02:78:de"],
"architecture": "x86_64"
},
"fields": {
"project": "jiankunking",
"type": "log"
}
}
}处理文本日志yml配置样例：
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/data/logs/*/*.log
fields:
project: jiankunking
multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d* '
multiline.negate: true
multiline.match: after
max_procs: 2
processors:
- add_host_metadata:
netinfo.enabled: true
setup.template.settings:
index.number_of_shards: 3
output.elasticsearch:
hosts: ["10.158.75.294:9200"]
pipeline: "timestamp-pipeline-id"
Configure the Elasticsearch output
多个pipeline配置（区分）也有
个人微信公众号：
作者：jiankunking 出处：http://blog.csdn.net/jiankunking