求帮助,解决TCP服务端CPU占用过高问题
自己写了一个TCP的server端,用来接收zip包,解压并存到mongo中,运行时发现CPU占用率一直居高不下,随着运行时间变多,每个端口下有174线程在运行,在代码内部添加close也不能解决,又没有大神清除,请问是什么原因,有什么解决办法??万分感谢
#!/usr/bin/python27
# coding:utf-8
import socketserver
import os
import subprocess
import time
import zipfile
import shutil
import configparser
import datetime
import pymongo
import json
from tool import *
import re
from read_execl import add_del_Dict
from stdDateTools import JsonTools
import redis
############配置文件中添加不同时间的数据库#####################
def initparse(fileName="somsCommun.ini"):
# 读取配置文件
cf = configparser.ConfigParser()
try:
cf.read(fileName)
dataStoreDir = cf.get("shorefileDir", "zipData")
tempDir = cf.get("shorefileDir", "jsontempData")
mongoIP = cf.get("shoreMongo", "ip")
mongoPort = cf.getint("shoreMongo", "port")
mongoDbName = cf.get("shoreMongo", "dataBase")
mongoCName = cf.get("shoreMongo", "collectionName")
# OneCName = cf.get("shoreMongo", "collectionOne")
runPort = cf.getint("serverRun", "port")
RedisIP = cf.get("shoreRedis", "ip")
RedisPort = cf.getint("shoreRedis", "port")
RedisDB = cf.get("shoreRedis", "db")
RedisPassword = cf.get("shoreRedis", "password")
# file_name = cf.get('xlsxName','file')
# adict = add_del_Dict(file_name)
except Exception as e:
print(e)
print("readConf Wrong")
# print(data_store_dir,shoreIP,shorePort,mongoIP,mongoPort,mongoDbName,mongoCName)
return (
dataStoreDir,
tempDir,
mongoIP,
mongoPort,
mongoDbName,
mongoCName,
runPort,
RedisIP,
RedisPort,
RedisDB,
RedisPassword)
# 存储zip包到各自日期下,解压后的json文件放到temp,存储到数据库后删除
class myserver(socketserver.BaseRequestHandler):
def handle(self):
print('connected...')
while True:
conn = self.request
conn.listen(5)
# 接收名称和size
pre_data = conn.recv(1024)
# print(pre_data)
authInfo = bytes.decode(pre_data, encoding="utf-8").strip()
if len(authInfo.strip()) < 5:
break
print(pre_data)
try:
file_name, file_size = authInfo.split(",")
n_file_name = file_name
n_file_name = n_file_name.split('_')[1]
try:
CName, jsonDir = to_dict(mongoCName, n_file_name)
JsonDir = dataToRecory_dir + jsonDir
mongoDB = pymongo.MongoClient(mongoIP, 27017)
dataBase = mongoDB[mongoDbName]
postsT = dataBase[CName]
OneCName = mongoCName + '_One'
postsO = dataBase[OneCName]
if postsT.find_one() is None:
try:
postsT.create_index('timestamp', unique=True)
except Exception as e:
print(e)
if postsO.find_one() is None:
try:
dataBase.create_collection(
OneCName, size=102400, capped=True, max=1)
except Exception as e:
print(e)
except Exception as e:
print('数据错误')
print(e)
except Exception as e:
print(e)
break
if file_size.isdigit() == False:
break
if int(file_size) < 50:
break
conn.sendall(bytes("sendData", encoding="utf-8"))
if not os.path.exists(dataStoreDir):
os.makedirs(dataStoreDir)
filePlace = os.path.join(dataStoreDir, file_name)
# print("filePlace : ",filePlace)
f = open(filePlace.encode(), 'wb')
flag = True
rec_size = 0
while flag:
if int(file_size) > rec_size:
data = conn.recv(1024)
rec_size += len(data)
else:
break
f.write(data)
# print('upload successed')
conn.sendall(bytes("YES", encoding="utf-8"))
f.close()
print('传输完成')
# 数据解压
try:
if not os.path.exists(JsonDir):
os.makedirs(JsonDir)
unzip_files(filePlace, JsonDir)
print("数据解压完成")
except Exception as e:
print(e)
break
# 获取解压后的文件路径
for root, dirs, files in os.walk(JsonDir):
for file in files:
unzipFile_path = os.path.join(JsonDir, file)
# print("unzipFile_path",unzipFile_path)
# 数据入库
try:
# 不依赖mongo版本
with open(unzipFile_path) as file_obj:
resJson = json.load(file_obj) # 返回列表数据,也支持字典
try:
if '_' not in CName:
postsO.insert_one(resJson)
r1 = redis.Redis(
host=RedisIP, port=RedisPort, db=RedisDB, password=RedisPassword)
try:
if r1.ping():
r1.hmset(mongoCName, resJson)
except Exception as e:
print(e)
except Exception as e:
print(e)
try:
# print(resJson)
#
# print(resJson['timestamp'])
postsT.insert_one(resJson)
# print(2)
except pymongo.errors.DuplicateKeyError as e:
print(
'索引重复,此次插入失败,重复索引为{}'.format(
resJson['timestamp']))
# print(3)
os.remove(unzipFile_path)
print('已删除:' + unzipFile_path)
break
except Exception as e:
print(e)
conn.close()
print("over")
if __name__ == '__main__':
class MyThreadingTCPServer(socketserver.ThreadingTCPServer):
"""重写socketserver.ThreadingTCPServer"""
# 服务停止后即刻释放端口,无需等待tcp连接断开
allow_reuse_address = True
dataStoreDir, dataToRecory_dir, mongoIP, mongoPort, mongoDbName, mongoCName, runPort, RedisIP, RedisPort, RedisDB, RedisPassword = initparse()
print(
dataStoreDir,
dataToRecory_dir,
mongoIP,
mongoPort,
mongoDbName,
mongoCName,
runPort)
instance = MyThreadingTCPServer(('0.0.0.0', runPort), myserver)
# poll_interval = 5
instance.serve_forever()