Datax hive 写 es
1、查看datax官网
https://github.com/alibaba/datax/tree/master
发现有 写es的案例
进入 case案例，查看 json
DataX/elasticsearchwriter.md at master · alibaba/DataX · GitHub
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
...
},
"writer": {
"name": "elasticsearchwriter",
"parameter": {
"endpoint": "http://xxx:9999",
"accessId": "xxxx",
"accessKey": "xxxx",
"index": "test-1",
"type": "default",
"cleanup": true,
"settings": {"index" :{"number_of_shards": 1, "number_of_replicas": 0}},
"discovery": false,
"batchSize": 1000,
"splitter": ",",
"column": [
{"name": "pk", "type": "id"},
{ "name": "col_ip","type": "ip" },
{ "name": "col_double","type": "double" },
{ "name": "col_long","type": "long" },
{ "name": "col_integer","type": "integer" },
{ "name": "col_keyword", "type": "keyword" },
{ "name": "col_text", "type": "text", "analyzer": "ik_max_word"},
{ "name": "col_geo_point", "type": "geo_point" },
{ "name": "col_date", "type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
{ "name": "col_nested1", "type": "nested" },
{ "name": "col_nested2", "type": "nested" },
{ "name": "col_object1", "type": "object" },
{ "name": "col_object2", "type": "object" },
{ "name": "col_integer_array", "type":"integer", "array":true},
{ "name": "col_geo_shape", "type":"geo_shape", "tree": "quadtree", "precision": "10m"}
]
}
}
}
]
}
}注意下：
有些参数不是必选的，可以不填
2、实测下：
{
"job": {
"setting": {
"speed": {
"channel": 1
}
},
"content": [
{
"reader": {
"name": "hdfsreader",
"parameter": {
"path": "/apps/hive/warehouse/071000/stg_hive_es_test/dt=2022-03-31/",
"defaultFS": "hdfs://sandbox-bigdata-1:8020/",
"column": [
{
"index": 0,
"type": "long"
},
{
"index": 1,
"type": "string"
},
{
"index": 2,
"type": "string"
},
{
"index": 3,
"type": "long"
},
{
"index": 4,
"type": "double"
},
{
"index": 5,
"type": "string"
},
{
"value": "2022-03-31",
"type": "string"
}
],
"fileType": "text",
"encoding": "UTF-8",
"fieldDelimiter": "\u0001"
}
},
"writer": {
"name": "elasticsearchwriter",
"parameter": {
"endpoint": "http://127.0.0.1:8200",
"index": "index_test_5",
"type": "rsl_bas_road_base",
"cleanup": false,
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 0
}
},
"discovery": false,
"batchSize": 1000,
"column": [
{
"name": "id",
"type": "long"
},
{
"name": "road_id",
"type": "keyword"
},
{
"name": "road_name",
"type": "keyword"
},
{
"name": "road_dir_no",
"type": "long"
},
{
"name": "flow",
"type": "double"
},
{
"name": "time",
"type": "date"
},
{
"name": "dt",
"type": "keyword"
}
]
}
}
}
]
}
}测试过程中，有个身份验证的错误注意下，在我的另一篇博客中。
datax hive 写 es 报错 Caused by: java.lang.IllegalArgumentException: Preemptive authentication set with_fyl005的博客-CSDN博客
source 端：
hive 数据 mock：
向 hive 分区表中插入数据_fyl005的博客-CSDN博客
sink 端：
es 端不需要提前创建 index的（不需要创建表），会自动创建表。
结果：
