#!/usr/bin/env python2.7
#-*- encoding: utf-8 -*-
import time
import datetime
import MySQLdb
import os
import re
import paramiko
# map的value值增量
def safe_inc_map_val(map, key, val):
if key in map.keys(): map[key] = map[key] + val
else: map[key] = val
# 当前时间
def time_now():
return datetime.datetime.now()
# 判断文件是否存在
def check_file_exists(filename):
if not os.path.exists(filename):
return False
return True
# 删除文件
def remove_file(file_name):
if(os.path.isfile(file_name)):
os.remove(file_name)
# 删除相应目录下的txt文件
def remove_text_file(filepath):
filelist = os.listdir(filepath)
for ele in filelist:
if ele.find('.txt')>-1:
os.remove(filepath+ele)
#IP转化为对应整数
def ip_to_int(ip):
int_val = reduce(lambda x,y:(x<<8)+y,map(int,ip.split('.')))
return int_val
#IP转化为对应整数
def Ip2Int(ip):
import struct,socket
return struct.unpack("!I",socket.inet_aton(ip))[0]
# 计算文本文件的行数
def line_count(file_name):
count = -1 #让空文件的行号显示0
for count,line in enumerate(open(file_name)): pass
#enumerate格式化成了元组,count就是行号,因为从0开始要+1
return count+1
# 使用外部系统程序 wc -l， 来计算文本文件的行数
def linecount_wc(file_name):
cmd = 'wc -l %s'%file_name
return int(os.popen(cmd).read().split()[0])
# 远程拷贝文件
def sftp_get(host_ip,remote_path,local_path,username,password):
t = paramiko.Transport((host_ip,22))
t.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(t)
src = remote_path
des = local_path
sftp.get(src,des)
t.close()
# 按行分割文本文件
def split_file(filepath, file_num):
total_rows = linecount_wc(filepath)
lines = total_rows/file_num+1 #这里大小加上1row，保证最后一个分割的文件不会丢数据
command = " split -l%d -a2 -d %s %s" % (lines, filepath, filepath) #其实使用的就是liunx的split
#print command
os.system(command)
# 按行分割文本文件，并将分割后的文件输出到新目录
def split_file2(filepath, new_filepath, file_num):
total_rows = linecount_wc(filepath)
lines = total_rows/file_num+1
command = " split -l%d -a2 -d %s %s" % (lines, filepath, new_filepath)
os.system(command)
# 返回结果集中的int值
def query_fetch_one_int(cursor, sql):
result = 0
n = cursor.execute(sql)
if (n > 0):
r = cursor.fetchone()
if r[0] != None:result = int(r[0])
return result
# 返回结果集中的float值
def query_fetch_one_float(cursor, sql):
result = 0
n = cursor.execute(sql)
if (n > 0):
r = cursor.fetchone()
if r[0] != None:result = float(r[0])
return result
# 返回结果集中的kv值列表
def query_fetch_pair_array(cursor, sql):
result = []
n = cursor.execute(sql)
if (n > 0):
ds = cursor.fetchall()
for r in ds:result.append((r[0], r[1]))
return result
# 返回结果集中的一组列表
def query_fetch_one_list(cursor , sql):
result = []
n = cursor.execute(sql)
if n>0:
ds = cursor.fetchall()
result = [ele[0] for ele in ds if ele!=None]
return result
