Spark 2.0 读写Elasticsearch 6

本地启动ES,我的版本是ES 6.3.2

• maven配置

---

 
        UTF-8
        2.1.2
        2.6.5
    

    
        
            org.apache.spark
            spark-sql_2.11
            ${spark.version}
        
        
            org.apache.spark
            spark-core_2.11
            ${spark.version}
        
        
            org.apache.spark
            spark-hive_2.11
            ${spark.version}
        

        
            org.apache.hadoop
            hadoop-common
            ${hadoop.version}
        
        
            org.elasticsearch
            elasticsearch-hadoop
            6.3.2
        
     
    

---

Java 代码如下:

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL;

/**
 * @author allen
 * @date 2018/5/9.
 */
public class SparkSqlToEs {

	public static void main(String[] args) {
		SparkSession spark=SparkSession.builder().appName("Sql2Es").config("es.nodes","localhost")
				.config("es.port","9200").master("local[3]").getOrCreate();

		Dataset person=spark.read().json("/opt/spark/examples/src/main/resources/people.json");

		person.show();

		JavaEsSparkSQL.saveToEs(person,"test/person");

	}
}

curl访问命令或者chrome打开

curl http://localhost:9200/test/person/_search?pretty

就可以看到新增加的数据document.

---

读es表

        Dataset table= JavaEsSparkSQL.esDF(spark,"trafficwisdom.hot_section/route");
        table.show();
        table.printSchema();