datax从mysql同步数据到elasticsearch（使用es的动态模板）

elasticsearch中设置动态模板

 

PUT _template/hkey_transferbill
{
  "index_patterns": "hkey_transferbill",
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 1
  },
  "mappings": {
    "data":{
      "dynamic_templates" : [ {  
        "keyword_field" : {  
          "unmatch" : "*Name",  
          "match_mapping_type" : "string",  
          "mapping" : {  
            "type" : "keyword"
          }  
        }  
      }, {  
        "text_fields" : {  
          "match" : "*Name",  
          "match_mapping_type" : "string",  
          "mapping" : {  
            "type" : "text",
            "fields": {
              "raw": {
                "type": "keyword"
              }
            }
          }  
        }  
      } ],  
      "properties": {
        "rowkey":
        {
          "type":"keyword"
        },
        "age":
        {
          "type":"keyword"
        }
      }
    }
  }
}

datax同步脚本 

{
  "job": {
    "setting": {
          "speed": {
            "byte": 8388608,
            "channel": 3
          },
          "errorLimit": {
            "record": 0,
            "percentage": 0.02
          }
        },
    "content": [
      {
        "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "username": "xxx",
                        "password": "xxx",
                        "column" : ["*"],                                   
                        "connection": [
                            {
                                "table": ["xxx"],
                                "jdbcUrl": [ "jdbc:mysql://xxx:3306/xxx" ]
                            }
                        ]
                    }                   
                },
        "writer": {
          "name": "elasticsearchwriter",
          "parameter": {
            "endpoint": "http://xxx:9200",
            "index": "myindex",
            "type": "data",
            "cleanup": true, #true表示插入前清空，即覆盖同步；false则追加同步
            "dynamic": true,  #这里一定要指定为true，否则使用的是datax的模板，而不会使用es的模板
            "settings": {"index" :{"number_of_shards": 2, "number_of_replicas": 1}},            
            "batchSize": 10000,
            "splitter": ",",
            "column": [
              {"name": "rowkey", "type": "id"},
              { "name": "userID","type": "keyword" },
              { "name": "userName","type": "text" }                     
            ]
          }
        }
      }
    ]
  }
}