logstash导入数据到Elasticsearch,安装logstash-input-jdbc插件

logstash是直接解压tar.gz文件,没有其它操作.

A.logstash安装logstash-input-jdbc插件

1. logstash-input-jdbc 是ruby开发的,先安装ruby

下载链接 https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.2.tar.gz

# tar -xvf ruby-2.7.2.tar.gz

# cd ruby-2.7.2

#  ./configure

# make && make install

这一步需要时间长,可以休息下.

输入

# ruby -v

出现版本号即ruby安装成功

2. 安装logstash-input-jdbc

进入logstash-x.x.x/bin目录

# ./logstash-plugin install logstash-input-jdbc

出现上面即logstash-input-jdbc安装成功.

B.增加配置文件

1. centOS配置

 

①. mysql.conf文件配置

input {
  stdin {
  }
  jdbc {
  jdbc_connection_string => "jdbc:mysql://localhost:3306/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
  # the user we wish to excute our statement as
  jdbc_user => "root"
  jdbc_password => mysql
  # the path to our downloaded jdbc driver  
  jdbc_driver_library => "F:/develop/maven/repository3/mysql/mysql-connector-java/5.1.41/mysql-connector-java-5.1.41.jar"
  # the name of the driver class for mysql
  jdbc_driver_class => "com.mysql.jdbc.Driver"
  jdbc_paging_enabled => "true"
  jdbc_page_size => "50000"
  #要执行的sql文件
  #statement_filepath => "/conf/course.sql"
  #时间以字符串的形式存入数据库,导出后,以UTC的形式转换为时间,实际的UTC时间需要加8小时才能一样.
  statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
  #定时配置
  schedule => "* * * * *"
  record_last_run => true
  last_run_metadata_path => "D:/Elastic/logstash-6.2.1/config/logstash_metadata"
  }
}


output {
  elasticsearch {
  #ES的ip地址和端口
  hosts => "localhost:9200"
  #hosts => ["localhost:9200","localhost:9202","localhost:9203"]
  #ES索引库名称
  index => "xc_course"
  document_id => "%{id}"
  document_type => "doc"
  template =>"D:/ElasticSearch/logstash-6.2.1/config/xc_course_template.json"
  template_name =>"xc_course"
  template_overwrite =>"true"
  }
  stdout {
 #日志输出
  codec => json_lines
  }
}

 

 

②. logstash_metadata文件配置

--- 2020-12-03 12:03:00.365800000 +08:00

 

③. xc_course_template.json配置

{
   "mappings" : {
      "doc" : {
         "properties" : {
            "charge" : {
               "type" : "keyword"
            },
            "description" : {
               "analyzer" : "ik_max_word",
               "search_analyzer" : "ik_smart",
               "type" : "text"
            },
            "end_time" : {
               "format" : "yyyy-MM-dd HH:mm:ss",
               "type" : "date"
            },
            "expires" : {
               "format" : "yyyy-MM-dd HH:mm:ss",
               "type" : "date"
            },
            "grade" : {
               "type" : "keyword"
            },
            "id" : {
               "type" : "keyword"
            },
            "mt" : {
               "type" : "keyword"
            },
            "name" : {
               "analyzer" : "ik_max_word",
               "search_analyzer" : "ik_smart",
               "type" : "text"
            },
            "pic" : {
               "index" : false,
               "type" : "keyword"
            },
            "price" : {
               "type" : "float"
            },
            "price_old" : {
               "type" : "float"
            },
            "pub_time" : {
               "format" : "yyyy-MM-dd HH:mm:ss",
               "type" : "date"
            },
            "qq" : {
               "index" : false,
               "type" : "keyword"
            },
            "st" : {
               "type" : "keyword"
            },
            "start_time" : {
               "format" : "yyyy-MM-dd HH:mm:ss",
               "type" : "date"
            },
            "status" : {
               "type" : "keyword"
            },
            "studymodel" : {
               "type" : "keyword"
            },
            "teachmode" : {
               "type" : "keyword"
            },
            "teachplan" : {
               "analyzer" : "ik_max_word",
               "search_analyzer" : "ik_smart",
               "type" : "text"
            },
            "users" : {
               "index" : false,
               "type" : "text"
            },
            "valid" : {
               "type" : "keyword"
            }
         }
      }
   },
   "template" : "xc_course"
}

这里的格式一定要和数据库匹配,代码转换时在yyyy前多加一个空格导致数据无法导入elasticsearch.

3. 下载mysql的驱动mysql-connector-java-8.0.18.jar放入mysql.conf文件指定的目录下:

jdbc_driver_library => "/usr/local/src/logstash-6.2.4/config/data/mysql-connector-java-8.0.18.jar"

2. elasticsearch配置

①. 在elasticsearch新建索引xc_course.

②. 设置映射,要和xc_course_template.json的映射一致,还要和数据库的列名一致.否则无法将数据库数据导入索引库.

POST  http://192.168.74.159:9200/xc_course/doc/_mapping

{
	"properties": {
		"description": {
			"analyzer": "ik_max_word",
			"search_analyzer": "ik_smart",
			"type": "text"
		},
		"grade": {
			"type": "keyword"
		},
		"id": {
			"type": "keyword"
		},
		"mt": {
			"type": "keyword"
		},
		"name": {
			"analyzer": "ik_max_word",
			"search_analyzer": "ik_smart",
			"type": "text"
		},
		"users": {
			"index": false,
			"type": "text"
		},
		"charge": {
			"type": "keyword"
		},
		"valid": {
			"type": "keyword"
		},
		"pic": {
			"index": false,
			"type": "keyword"
		},
		"qq": {
			"index": false,
			"type": "keyword"
		},
		"price": {
			"type": "float"
		},
		"price_old": {
			"type": "float"
		},
		"st": {
			"type": "keyword"
		},
		"status": {
			"type": "keyword"
		},
		"studymodel": {
			"type": "keyword"
		},
		"teachmode": {
			"type": "keyword"
		},
		"teachplan": {
			"analyzer": "ik_max_word",
			"search_analyzer": "ik_smart",
			"type": "text"
		},
		"expires": {
			"type": "date",
			"format": "yyyy‐MM‐dd HH:mm:ss"
		},
		"pub_time": {
			"type": "date",
			"format": "yyyy‐MM‐dd HH:mm:ss"
		},
		"start_time": {
			"type": "date",
			"format": "yyyy‐MM‐dd HH:mm:ss"
		},
		"end_time": {
			"type": "date",
			"format": "yyyy‐MM‐dd HH:mm:ss"
		}
	}
}

数据库列

C.运行logstash

进入logstash下的bin目录

输入命令:

# ./logstash -f ../config/mysql.conf