ES 在HIVE中创建表存储数据

所有操作使用shell脚本方式执行

1. 创建指定mapping文件

more ./create/es_reg_user_goods_article_recommend_mapping.json

{
                "et_reg_user_goods_article_recommend" : {
                        "properties" : {
                                "user_id" : {"type" : "long", "store": "yes"},
                                "article_ids" : {"type" : "string", "store": "yes", "index": "not_analyzed"},
                                "create_date" : {"type" : "string", "store": "yes", "index": "not_analyzed"}
                        }
                }
}

2. 在ES中创建索引

# del index 
# 删除索引
curl -XDELETE http://192.168.1.1:9200/es_reg_user_article_recommend

# create index
# 创建索引
curl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/ -d'
{
        "mappings" : {
                "_default_" : {
                        "dynamic" : "strict"
                }
        }
}'

# create alias
# 创建别名
curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'
{
        "actions" : [{ "add" : { "index" : "es_reg_user_goods_article_recommend", "alias" : "reg_user_goods_article_recommend_index"}}]
}'

# modify alias
# 修改别名
curl -XPOST 'http://192.168.1.1:9200/_aliases' -d'
{
        "actions" : [
                { "remove" : { "index" : "es_reg_user_article_recommend", "alias" : "reg_user_article_recommend_index" }},
                { "add" : { "index" : "es_reg_user_article_recommend_new", "alias" : "reg_user_article_recommend_index"}}
        ]
}'

# create mapping
# 创建mapping
curl -XPUT http://192.168.1.1:9200/es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend/_mapping?pretty -d @/opt/DATA/goldmine/src/mining/user_recommend_mining/goods_article_recommend/create/es_reg_user_goods_article_recommend_mapping.
json

3. 创建 HIVE 表

more ./create/hive_to_es_reg_user_goods_article_recommend.sh


#!/bin/sh
###
# 1. del jar
hadoop fs -rm /user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar

# 2. put jar
hadoop fs -put /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jar /user/hive/warehouse/jars/

# 3. create index
sh create_es_reg_user_article_recommend_mapping

###
#--org.apache.commons.httpclient_3.1.0.v201012070820.jar这个jar包一般是不用加的，除非在报httpclient相关错误的时候才加
#add jar hdfs:/opt/data/jar/org.apache.commons.httpclient_3.1.0.v201012070820.jar;\

# 4. create table
# /usr/local/goldmine/hive/default/dg-lib/elasticsearch-hadoop-2.4.1.jar
reg_user_goods_article_recommend_create_table="add jar hdfs:/user/hive/warehouse/jars/elasticsearch-hadoop-2.4.1.jar;\
        create external table if not exists des_reg_user_goods_article_recommend(
                user_id bigint,
                article_ids string,
                create_date string
        ) 
        STORED BY 'org.elasticsearch.hadoop.hive.EsStorageHandler' 
        TBLPROPERTIES(
                'es.nodes'='192.168.1.1:9200',
                'es.resource' = 'es_reg_user_goods_article_recommend/et_reg_user_goods_article_recommend',
                'es.index.auto.create'='false',
                'es.mapping.id'='user_id',
                'es.mapping.names'='article_ids:article_ids,create_date:create_date');"
#       --es.resource斜线前面为索引，斜线后面为mapping名称。mapping可以不用事先建好，当数据插入时会自动根据数据类型创建mapping。

hive -e "${reg_user_goods_article_recommend_create_table}"