有用的python脚本_几个有用的python脚本

1、通过pythony连接Hive执行Hql的脚本

[spark@Master Py_logproc]$ cat py2hive.py

#!/usr/bin/env python

import sys

sys.path.append('/home/spark/opt/hive-1.2.1/lib/py')

from hive_service import ThriftHive

from hive_service.ttypes import HiveServerException

from thrift import Thrift

from thrift.transport import TSocket

from thrift.transport import TTransport

from thrift.protocol import TBinaryProtocol

def hiveExe(sql):

try:

transport = TSocket.TSocket('127.0.0.1', 10000)

transport = TTransport.TBufferedTransport(transport)

protocol = TBinaryProtocol.TBinaryProtocol(transport)

client = ThriftHive.Client(protocol)

transport.open()

client.execute(sql)

print "The return value is : "

print client.fetchAll()

print "............"

transport.close()

except Thrift.TException, tx:

print '%s' % (tx.message)

if __name__ == '__main__':

select_sql="select url_current from yemao1_log limit 100";

hiveExe(select_sql)

2、通过python解析json数据并导出成普通文本的脚本

[spark@Master Py_logproc]$ cat json2file.py

# -*- encoding:utf-8 -*-

import json

import warnings

warnings.filterwarnings("ignore")

if __name__=="__main__":

import sys

reload(sys)

sys.setdefaultencoding( "utf-8" )

if len(sys.argv)==1:

print "need argv"

else:

print sys.argv

for json_array in open('/home/spark/opt/Log_Data/Py_logproc/log_tmpdir/yemaopythonlog'):

yemao_array = json.loads(json_array)

for yemao in yemao_array:

if not yemao.has_key('_reason'):

time = yemao['time']

url_current = yemao['url_current']

phone = yemao['phone']

token = yemao['token']

f=open('/home/spark/opt/Log_Data/Py_logproc/log_tmpdir/xxxx.txt','a+')

f.write('%d'%time+'|'+url_current+'|'+phone+'|'+token+'\n')

f.close()

print 'yemao_array_insert_file done'

3、Mysql连接供调用的python脚本

[spark@Master Py_logproc]$ cat db.py

import MySQLdb

db_config = {

'host': '120.55.189.188',

'user': 'datahouse',

'passwd': 'DTHS2016',

'port': 3306,

'db': 'logdata',

'charset': 'utf8'

}

def getDB():

try:

conn = MySQLdb.connect(host=db_config['host'],user=db_config['user'],passwd=db_config['passwd'],port=db_config['port'],charset=db_config['charset'])

conn.autocommit(True)

curr = conn.cursor()

curr.execute("SET NAMES utf8");

curr.execute("USE %s" % db_config['db']);

return conn, curr

except MySQLdb.Error,e:

print "Mysql Error %d: %s" % (e.args[0], e.args[1])

return None, None