使用logstash读取postgis数据库导入Elasticsearch，并配置模板映射。

首先将postgresql驱动程序(jar包)拷贝到linux，然后编写PGtoES.conf和postgis_test.json，进入logstash的bin目录，执行命令：

./logstash -f logstash_pg_test/PGtoES.conf

如果不成功需要将/root/opt/logstash-6.3.0/data删除rm –rf ../data

 

 

PGtoES.conf

input{  
    stdin{
       
    }
    jdbc{  
        jdbc_connection_string => "jdbc:postgresql://{IP}:5432/gansu"  # jdbc数据库连接
        jdbc_user => "postgres"  
        jdbc_password => "postgres"  
        jdbc_driver_library => "/root/opt/logstash-6.3.0/bin/logstash_postgis/postgresql-42.1.1.jar"  # postgresql jar包文件路径
        jdbc_driver_class => "org.postgresql.Driver"  # jdbc数据库驱动
        jdbc_paging_enabled => "true"  
        jdbc_page_size => "50000"  # 同步数据分页设置
        lowercase_column_names => "false"  #是否将 column 名称转小写，如果不写，默认将字段转成小写，false则不转小写
        #use_column_value => false  #使用其它字段追踪，而不是用时间
        #tracking_column => "UPDATETIME"
        #statement_filepath => "/root/opt/logstash/bin/logstash_jdbc_test/jdbc.sql"  # 需同步的数据执行的SQL文件路径
        statement => '  select gid as "DZZYID","DZMC" as "DZZYMC",s."DZMC" as "DZZYMC_PINYIN","FZR_XM","XGR_XM","PY","MC","CSLXMC","XGSJ" as "UPDATETIME","DZJDZB" as "X","DZWDZB" as "Y",ST_AsText(geom) as geom  from testpoint s ' # 也可以直接将sql配置在这里
        #, s."DZWDZB" ||E','||  s."DZJDZB"  as "LOCATION"   #这个不成功，单引号转义不成功
        #schedule => "* * * * *"   # 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
        type => "postgis_test"
    }
}

filter {
    mutate {
       add_field => {"DATATYPE"=>"PT"}  
       add_field => {"LAYER"=>"postgis_test"}  
       add_field => {"LOCATION" => "%{Y},%{X}"}
       remove_field => "@version"
       remove_field => "@timestamp"   
    }
    date {
            match => ["UPDATETIME", "yyyyMMdd"] 
            target => "UPDATETIME"    
    }
    if [UPDATETIME] {
        ruby { 
            code => "event.set('UPDATETIME', event.get('UPDATETIME').time.localtime + 8*60*60)" 
        }
    }
    
}

output {
    #stdout{
        #codec => json_lines
    #}
    if [type] == "postgis_test"{
        elasticsearch{  
            hosts => ["172.25.16.154:9200", "172.25.16.147:9200", "172.25.16.148:9200"]  # ES连接
            index => "postgis_test"      # ES索引名称
            document_id => "%{DZZYID}"     #_id字段与 DZZYID 字段值一样，以便按照ID查询，_id是ES自带的字段，不能在input.statement中输入
            #document_type => "doc"
            manage_template => true
            template_overwrite => true
            template_name => "postgis_test"
            template => "/root/opt/logstash-6.3.0/bin/logstash_postgis/postgis_test.json"
            codec => json_lines
        } 
    }
}

模板映射文件：postgis_test.json

{
"template": "postgis_test",
"mappings": {
"doc": {
"properties": {
"DZZYID": {
"type": "keyword"
},
"LAYER": {
"type": "keyword"
},
"DZZYMC": {
"type": "text",
"term_vector": "with_positions_offsets",
"analyzer": "ik_max_word"
},
"DZZYMC_PINYIN": {
"type": "text",
"term_vector": "with_positions_offsets",
"analyzer": "pinyin"
},
"LOCATION": {
"type": "geo_point"
},
"X": {
"type": "keyword"
},
"Y": {
"type": "keyword"
},
"UPDATETIME": {
"type": "date"
},
"DATATYPE": {
"type": "keyword"
},
"CSLXMC": {
"type": "keyword"
},
"PY": {
"type": "keyword"
},
"XGR_XM": {
"type": "keyword"
},
"FZR_XM": {
"type": "keyword"
}
}
}
}
}