first python script

• 背景：刚好公司有一个刷数据的任务，就上手了python，早该开始了。
• 小结知识点如下：
• 1 python的默认编码是unicode，如果在程序中输入中文就会有乱码，如果文件中中文是utf-8编码的，就可以用以下这种方式解码。

import os
import codecs
path = os.getcwd();
print(path);
os.chdir('/Users/liuchaoqun01/Desktop')
print(os.getcwd());
with codecs.open('to see you', encoding='utf-8') as f:
    print(f.read()) 

• 2 cc助攻的解码问题：文本文件是latin1编码的中文，120w行。甚至蛋疼，一直不知道应该怎么转为utf-8。经cc点化：

with open('d:/res.txt','wb') as w:
    with open(r'd:/entity.list.all.txt','rb') as f:
        for x in f:
            try:
                w.write(x.decode('gb2312').encode('utf8'))
            except Exception:
                w.write(b'\n')

还是一知半解，等待cc指正！

• 3 这是最后成形的代码，本质上就是一个数据脚本，总共跑了1hour，120w数据，其实最后发现120w的数据查询太耗时了，一开始就应该用批量查询！！

import sys,re,os
import datetime
import MySQLdb
begin = datetime.datetime.now()
reload(sys)
sys.setdefaultencoding("utf-8")
path = os.getcwd();
os.chdir('/Users/liuchaoqun01/Documents/cloud/input')
predictFile = open('predict.trade2.entity.list.all', 'r')
count = 0
businessList = []
emptyList = []
predictFileList = []
resultList = []
emptyList = []
with open('res.txt','r') as w:
     for business in w:
          line = business.replace('\n', '')
          businessList.append(line)
for predict in predictFile:
     p = re.compile(r'\s|__')
     ss = p.split(predict)
     if len(ss) < 3:
          count = count + 1
          line = '-1'
     else:
          line = "select industryname from aodfeed.industry where industryid = " + (ss[2])
     predictFileList.append(line)
     line = ''
predictFile.close()
print 'The count of blank line is:' + str(count)

# mysql
outputResult = open('/Users/liuchaoqun01/Documents/cloud/output/result', 'w+')
outputEmpty = open('/Users/liuchaoqun01/Documents/cloud/output/empty', 'w+')
conn = MySQLdb.connect(host='10.99.196.227', port=8306, user='test', passwd='xxx', db='xxx', charset='utf8')
cursor = conn.cursor()
for i in range(len(predictFileList)):
     predict = predictFileList[i]
     business = businessList[i]
     if predict == '-1':
          emptyList.append(business + '\t' + '-1')
     else:
          sql = predict
          cursor.execute(sql)
          r = cursor.fetchall()
         # print (business + '\t' + r[0][0].decode('utf-8'))
          resultList.append(business + '\t' + r[0][0])

# write
for out in resultList:
     outputResult.write(out + '\n')
outputResult.close()

for out in emptyList:
     outputEmpty.write(out + '\n')
outputEmpty.close()
print 'process end!'
end = datetime.datetime.now()
print (end - begin)