话不多说直接上代码
/*********************************** 写数据到ElasticSearch ***************************************/
//从配置文件中获取es的地址
ListhttpHosts =new ArrayList<>();
httpHosts.add(new HttpHost(parameterTool.getRequired("ELASTICSEARCH_HOST"),9200,"http"));
/*//从配置文件中读取 bulk flush size，代表一次批处理的数量int bulkSize = parameterTool.getInt("ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS", 40);*/
// 创建elasticsearch Sink
ElasticsearchSink.BuilderesSinkBuilder =new ElasticsearchSink.Builder<>(
httpHosts,
new ElasticsearchSinkFunction() {
public IndexRequest createIndexRequest(String element) {
Mapjson =new HashMap<>();
String line =element.substring(element.split("\\|\\^\\|")[0].length() +3);
KafkaEvent kafkaEvent =new KafkaEvent().fromString(line);
json.put("dts_id",kafkaEvent.getDid().toString());
json.put("business_time",kafkaEvent.getBt());
json.put("protocol_id",kafkaEvent.getPi());
json.put("user_view_status",kafkaEvent.getUvs().longValue());
return Requests.indexRequest()
.id(element.split("\\|\\^\\|")[0])
.index(parameterTool.getRequired("ES_INDEX_NAME"))
.type(parameterTool.getRequired("ES_INDEX_TYPE"))//ES_INDEX_TYPE
.source(json);
}
@Override
public void process(String element,RuntimeContext ctx,RequestIndexer indexer) {
indexer.add(createIndexRequest(element));
}
}
);
//批处理最大数
esSinkBuilder.setBulkFlushMaxActions(parameterTool.getInt("ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS",40));
esSinkBuilder.setRestClientFactory(
new RestClientFactory() {
@Override
public void configureRestClientBuilder(RestClientBuilder restClientBuilder) {
final CredentialsProvider credentialsProvider =new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(parameterTool.getRequired("ELASTICSEARCH_NAME"),
parameterTool.getRequired("ELASTICSEARCH_PASSWD")));
restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {//设置自定义http客户端配置
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
})/*.setMaxRetryTimeoutMillis(2000)*/;
}
}
);
//将去重后的数据写入到ElasticSearch中
resulted.addSink(esSinkBuilder.build());