squid 日志分析 - hive 删除

 #!/usr/bin/env python

#-*-coding:utf8-*-

 

import os,re,threading,tarfile,shutil,time,sys,datetime

import sys,traceback

sys.path.append('/usr/local/hive/lib/py')

from thrift import Thrift

from thrift.transport import TSocket

from thrift.transport import TTransport

from thrift.protocol import TBinaryProtocol

from hive_service import ThriftHive

from hive_service.ttypes import HiveServerException

 

transport = TSocket.TSocket('127.0.0.1',10000)

transport = TTransport.TBufferedTransport(transport)

protocol = TBinaryProtocol.TBinaryProtocol(transport)

client = ThriftHive.Client(protocol)

transport.open()

client.execute("use squid")

 

 

class threads_del(threading.Thread):

    def __init__(self,pt_ip,pt_dt):

        threading.Thread.__init__(self)

        self.pt_ip = pt_ip

        self.pt_dt = pt_dt

 

    def run(self):

        client.execute("set hive.exec.dynamic.partition=true")

        client.execute("set hive.exec.dynamic.partition.mode=nonstrict")

        sql = "alter table squid_log drop partition(pt_ip = '%s', pt_dt=%s)"%(self.pt_ip,self.pt_dt)

        client.execute(sql)

 

def work():

    client.execute('show partitions squid_log')

    info = client.fetchAll()

    infos = {}

    for i in info:

        a = re.split("/",i)

        try:

            infos[re.split("=",a[0])[1]].append(re.split("=",a[1])[1])

        except:

            infos[re.split("=",a[0])[1]] = []

            infos[re.split("=",a[0])[1]].append(re.split("=",a[1])[1])

 

    for k,v in infos.items():

        st = threads_del(k,min(v))

        st.start()

        st.join()

 

    while threading.active_count() >1:

        time.sleep(1)

 

    transport.close()

 

 

if __name__ == "__main__":

    st = work()

 

本文出自 “欢迎评论，欢迎点赞” 博客，谢绝转载！