几个有用的python脚本

1、通过pythony连接Hive执行Hql的脚本
[spark@Master Py_logproc]$ cat py2hive.py 

#!/usr/bin/env python
import sys
sys.path.append('/home/spark/opt/hive-1.2.1/lib/py')
from hive_service import ThriftHive
from hive_service.ttypes import HiveServerException
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
def hiveExe(sql):
    try:
        transport = TSocket.TSocket('127.0.0.1', 10000) 
        transport = TTransport.TBufferedTransport(transport)
        protocol = TBinaryProtocol.TBinaryProtocol(transport)
        client = ThriftHive.Client(protocol)
        transport.open()
        client.execute(sql)
        print "The return value is : " 
        print client.fetchAll()
        print "............"
        transport.close()
    except Thrift.TException, tx:
        print '%s' % (tx.message)
if __name__ == '__main__':
    select_sql="select url_current from yemao1_log limit 100";
    hiveExe(select_sql)

   
2、通过python解析json数据并导出成普通文本的脚本
[spark@Master Py_logproc]$ cat json2file.py

# -*- encoding:utf-8 -*-
import json
import warnings
warnings.filterwarnings("ignore")




if __name__=="__main__": 
 import sys
 reload(sys)
 sys.setdefaultencoding( "utf-8" ) 
 
 if len(sys.argv)==1:  
     print "need argv"  
 else:  
     print sys.argv  




for json_array in open('/home/spark/opt/Log_Data/Py_logproc/log_tmpdir/yemaopythonlog'):
  yemao_array = json.loads(json_array)
  for yemao in yemao_array: 
   if not yemao.has_key('_reason'):
    
    time            =   yemao['time']
    url_current     =   yemao['url_current']
    phone           =   yemao['phone']
    token           =   yemao['token']
    
    f=open('/home/spark/opt/Log_Data/Py_logproc/log_tmpdir/xxxx.txt','a+')
    f.write('%d'%time+'|'+url_current+'|'+phone+'|'+token+'\n')
    f.close()




print 'yemao_array_insert_file done'

3、Mysql连接供调用的python脚本
[spark@Master Py_logproc]$ cat db.py

import MySQLdb




db_config = {
    'host': '120.55.189.188',
    'user': 'datahouse',
    'passwd': 'DTHS2016',
    'port': 3306,
    'db': 'logdata',
    'charset': 'utf8'
}




def getDB():
  try:
    conn = MySQLdb.connect(host=db_config['host'],user=db_config['user'],passwd=db_config['passwd'],port=db_config['port'],charset=db_config['charset'])
    conn.autocommit(True)
    curr = conn.cursor()
    curr.execute("SET NAMES utf8");
    curr.execute("USE %s" % db_config['db']);




    return conn, curr
  except MySQLdb.Error,e:
    print "Mysql Error %d: %s" % (e.args[0], e.args[1])
    return None, None