hive升级至0.11 partition schemal问题

  线上由hive0.10升级到hive0.11之后，部分etl job运行出现问题，具体表现为：

在分区表中，当改变table的表的字段类型之后，旧的partition数据可能会出现NULL的情况。

这其实是hive0.11的一个新的特性，每个partition可以有自己的schemal信息。

分析hive0.10和hive0.11 mysql中的元数据信息，发现在table的schemal更改后，两种版本下新的partition都会继承table的schemal，而老的partition不会改变。

在hive0.11这个特性的描述中：

Rows in partitions are now read using partition schema and than made to comply with table schema, instead of being read directly using table schema.

也就是说，hive0.11的partition都是直接读自己的column_v2表中的信息，而hive0.10是读table的信息，这个特性是默认开启的，这就造成在升级到hive0.11之后部分job运行结果有问题。

解决方法：

通过更新mysql中hive元数据的sds表，把partition的cd_id信息都改成对应的table的cd_id的值即可。

实际操作中需要做3个表的join相关操作，在mysql中计算会比较慢，因此把数据load到gp中进行计算，通过python生成update语句做更新(因为涉及到元数据的更新，暂时还是用了手动的方式来更新，另外做了一个监控，当有数据不一致时会有报警邮件产生)：

python代码如下：

import psycopg2
import time,sys
class gpUtil:
        def __init__(self,host,user,passwd,db):
                self.host = host
                self.user = user
                self.passwd = passwd
                self.db = db
        def db_connect(self):
                self.conn = psycopg2.connect(host=self.host,user=self.user,password=self.passwd,database=self.db)
                self.cursor = self.conn.cursor()
        def db_fetch_all(self,sql):
                self.cursor.execute(sql)
                try:
                        result = self.cursor.fetchall()
                except:
                        result = "Wrong"
                return result
        def db_close(self):
                self.cursor.close()
                self.conn.close()
def get_list(sql):
        conn = gpUtil('xxxx','xxxxx','xxxxx','xxxxxx')
        conn.db_connect()
        host_list = conn.db_fetch_all(sql)
        return host_list
        conn.db_close()
def get_item_list():
                tble_list = []
                d = 0
                sql = """
                SELECT TBL_NAME,a.TBL_ID,TABLE_CD_ID,PART_NAME,PART_ID,PARTITION_CD_ID,PARITION_SD_ID FROM                                        
(select PART_ID,PART_NAME,TBL_ID,CD_ID AS PARTITION_CD_ID,b.SD_ID  AS PARITION_SD_ID from vipbi.partitions a,vipbi.sds b where a.SD_ID=b.SD_ID)             
a join (select TBL_ID,TBL_NAME,CD_ID AS TABLE_CD_ID from vipbi.tbls a,vipbi.sds b where a.SD_ID=b.SD_ID) b on a.TBL_ID=b.TBL_ID and a.PARTITION_CD_ID<>b.TABLE_CD_ID
                """
                re = get_list(sql)
                for line in re:
                        line = list(line)
                        table_name = line[0]
                        table_id = line[1]
                        table_cd_id = line[2]
                        part_name = line[3]
                        part_id = line[4]
                        part_cd_id = line[5]
                        part_sd_id =  line[6]
                        tble_list.append([table_name,table_id,table_cd_id,part_name,part_id,part_cd_id,part_sd_id])
                return tble_list
def change_partition(tble_list):
                if len(tble_list) == 0:
                        print "no need to update"
                        sys.exit(1)
                now = time.time()
                change_content = ''
                rollback_content =  ''
                change_sql = open("/home/hdfs/bin/hadoop_tools/sql/meta_change" + str(int(now)) + ".sql",'w+')
                rollback_sql = open("/home/hdfs/bin/hadoop_tools/sql/meta_rollback"  + str(int(now)) + ".sql",'w+')
                for table_item in tble_list:
                        table_name = table_item[0]
                        table_id = table_item[1]
                        table_cd_id = table_item[2]
                        part_name = table_item[3]
                        part_id = table_item[4]
                        part_cd_id = table_item[5]
                        part_sd_id = table_item[6]
                        change_content += """-- table para<name:%s,id:%s,cd_id:%s>; partition para<name:%s,id:%s,cd_id:%s,sd_id:%s>\n""" % (table_name,table_id,table_cd_id,part_name,part_id,part_cd_id,part_sd_id)
                        rollback_content += """-- table para<name:%s,id:%s,cd_id:%s>; partition para<name:%s,id:%s,cd_id:%s,sd_id:%s>\n""" % (table_name,table_id,table_cd_id,part_name,part_id,part_cd_id,part_sd_id)
                        change_content += """ update sds set CD_ID='%s' where SD_ID='%s';\n"""  % (table_cd_id,part_sd_id)
                        rollback_content += """ update sds set CD_ID='%s' where SD_ID='%s';\n"""  % (part_cd_id,part_sd_id)
                change_sql.write(change_content)
                rollback_sql.write(rollback_content)
                change_sql.close()
                rollback_sql.close()
if __name__ == '__main__':
                #change_content = ''
                start_time = time.time()
                tble_list = []
                tble_list = get_item_list()
                change_partition(tble_list)
                stop_time = time.time()
                elapse_time = stop_time - start_time
                print "elapse_time is  %s" % (str(elapse_time))