Spark读取mongoDB数据写入Hive普通表和分区表

版本：
spark 2.2.0
hive 1.1.0
scala 2.11.8
hadoop-2.6.0-cdh5.7.0
jdk 1.8
MongoDB 3.6.4

一 原始数据及Hive表
MongoDB数据格式

{
    "_id" : ObjectId("5af65d86222b639e0c2212f3"),
    "id" : "1",
    "name" : "lisi",
    "age" : "18",
    "deptno" : "01"
}

Hive普通表

create table mg_hive_test(
id string,
name string,
age string,
deptno string
)row format delimited fields terminated by '\t';

Hive分区表

create table  mg_hive_external(
id string,
name string,
age string
)
partitioned by (deptno string)
row format delimited fields terminated by '\t';

二 IDEA+Maven+Java
依赖

 <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-sql_2.11artifactId>
      <version>${spark.version}version>
    dependency>
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-hive_2.11artifactId>
      <version>${spark.version}version>
    dependency>
    <dependency>
      <groupId>org.mongodbgroupId>
      <artifactId>mongo-java-driverartifactId>
      <version>3.6.3version>
    dependency>
    <dependency>
      <groupId>org.mongodb.sparkgroupId>
      <artifactId>mongo-spark-connector_2.11artifactId>
      <version>2.2.2version>
    dependency>

代码

package com.huawei.mongo;/*
 * @Author: Create by Achun
 *@Time: 2018/6/2 21:00
 *
 */

import com.mongodb.spark.MongoSpark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.bson.Document;

import java.io.File;
import java.util.ArrayList;
import java.util.List;

public class sparkreadmgtohive {
    public static void main(String[] args) {
        //spark 2.x
        String warehouseLocation = new File("spark-warehouse").getAbsolutePath();
        SparkSession spark = SparkSession.builder()
                .master("local[2]")
                .appName("SparkReadMgToHive")
                .config("spark.sql.warehouse.dir", warehouseLocation)
                .config("spark.mongodb.input.uri", "mongodb://127.0.0.1:27017/test.mgtest")
                .enableHiveSupport()
                .getOrCreate();
        JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());

        //spark 1.x
//        JavaSparkContext sc = new JavaSparkContext(conf);
//        sc.addJar("/Users/mac/zhangchun/jar/mongo-spark-connector_2.11-2.2.2.jar");
//        sc.addJar("/Users/mac/zhangchun/jar/mongo-java-driver-3.6.3.jar");
//        SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("SparkReadMgToHive");
//        conf.set("spark.mongodb.input.uri", "mongodb://127.0.0.1:27017/test.mgtest");
//        conf.set("spark. serializer","org.apache.spark.serializer.KryoSerialzier");
//        HiveContext sqlContext = new HiveContext(sc);
//        //create df from mongo
//        Dataset df = MongoSpark.read(sqlContext).load().toDF();
//        df.select("id","name","name").show();

        String querysql= "select id,name,age,deptno,DateTime,Job from mgtable b";
        String opType ="P";

        SQLUtils sqlUtils = new SQLUtils();
        List column = sqlUtils.getColumns(querysql);

        //create rdd from mongo
        JavaRDD rdd = MongoSpark.load(sc);
        //将Document转成Object
        JavaRDD