求帮助,解决TCP服务端CPU占用过高问题

求帮助,解决TCP服务端CPU占用过高问题
自己写了一个TCP的server端,用来接收zip包,解压并存到mongo中,运行时发现CPU占用率一直居高不下,随着运行时间变多,每个端口下有174线程在运行,在代码内部添加close也不能解决,又没有大神清除,请问是什么原因,有什么解决办法??万分感谢

#!/usr/bin/python27
# coding:utf-8

import socketserver
import os
import subprocess
import time
import zipfile
import shutil
import configparser
import datetime
import pymongo
import json
from tool import *
import re
from read_execl import add_del_Dict
from stdDateTools import JsonTools
import redis

############配置文件中添加不同时间的数据库#####################


def initparse(fileName="somsCommun.ini"):
    #    读取配置文件
    cf = configparser.ConfigParser()
    try:
        cf.read(fileName)
        dataStoreDir = cf.get("shorefileDir", "zipData")
        tempDir = cf.get("shorefileDir", "jsontempData")

        mongoIP = cf.get("shoreMongo", "ip")
        mongoPort = cf.getint("shoreMongo", "port")
        mongoDbName = cf.get("shoreMongo", "dataBase")
        mongoCName = cf.get("shoreMongo", "collectionName")
        # OneCName = cf.get("shoreMongo", "collectionOne")

        runPort = cf.getint("serverRun", "port")

        RedisIP = cf.get("shoreRedis", "ip")
        RedisPort = cf.getint("shoreRedis", "port")
        RedisDB = cf.get("shoreRedis", "db")
        RedisPassword = cf.get("shoreRedis", "password")
        # file_name = cf.get('xlsxName','file')
        # adict = add_del_Dict(file_name)

    except Exception as e:
        print(e)
        print("readConf Wrong")
    # print(data_store_dir,shoreIP,shorePort,mongoIP,mongoPort,mongoDbName,mongoCName)
    return (
        dataStoreDir,
        tempDir,
        mongoIP,
        mongoPort,
        mongoDbName,
        mongoCName,
        runPort,
        RedisIP,
        RedisPort,
        RedisDB,
        RedisPassword)


# 存储zip包到各自日期下,解压后的json文件放到temp,存储到数据库后删除
class myserver(socketserver.BaseRequestHandler):

    def handle(self):

        print('connected...')
        while True:
            conn = self.request
            conn.listen(5)
            # 接收名称和size
            pre_data = conn.recv(1024)
            # print(pre_data)
            authInfo = bytes.decode(pre_data, encoding="utf-8").strip()

            if len(authInfo.strip()) < 5:
                break
            print(pre_data)
            try:
                file_name, file_size = authInfo.split(",")
                n_file_name = file_name
                n_file_name = n_file_name.split('_')[1]

                try:
                    CName, jsonDir = to_dict(mongoCName, n_file_name)
                    JsonDir = dataToRecory_dir + jsonDir
                    mongoDB = pymongo.MongoClient(mongoIP, 27017)
                    dataBase = mongoDB[mongoDbName]
                    postsT = dataBase[CName]
                    OneCName = mongoCName + '_One'
                    postsO = dataBase[OneCName]

                    if postsT.find_one() is None:
                        try:
                            postsT.create_index('timestamp', unique=True)
                        except Exception as e:
                            print(e)

                    if postsO.find_one() is None:
                        try:
                            dataBase.create_collection(
                                OneCName, size=102400, capped=True, max=1)
                        except Exception as e:
                            print(e)

                except Exception as e:
                    print('数据错误')
                    print(e)
            except Exception as e:
                print(e)
                break

            if file_size.isdigit() == False:
                break
            if int(file_size) < 50:
                break

            conn.sendall(bytes("sendData", encoding="utf-8"))

            if not os.path.exists(dataStoreDir):
                os.makedirs(dataStoreDir)
            filePlace = os.path.join(dataStoreDir, file_name)
            # print("filePlace : ",filePlace)
            f = open(filePlace.encode(), 'wb')
            flag = True
            rec_size = 0
            while flag:
                if int(file_size) > rec_size:
                    data = conn.recv(1024)
                    rec_size += len(data)
                else:
                    break
                f.write(data)
            # print('upload successed')
            conn.sendall(bytes("YES", encoding="utf-8"))
            f.close()

            print('传输完成')

            # 数据解压
            try:
                if not os.path.exists(JsonDir):
                    os.makedirs(JsonDir)
                unzip_files(filePlace, JsonDir)
                print("数据解压完成")
            except Exception as e:
                print(e)
                break
            # 获取解压后的文件路径
            for root, dirs, files in os.walk(JsonDir):
                for file in files:
                    unzipFile_path = os.path.join(JsonDir, file)
#                    print("unzipFile_path",unzipFile_path)
                    # 数据入库
                    try:
                        # 不依赖mongo版本
                        with open(unzipFile_path) as file_obj:
                            resJson = json.load(file_obj)  # 返回列表数据,也支持字典

                        try:
                            if '_' not in CName:
                                postsO.insert_one(resJson)
                                r1 = redis.Redis(
                                    host=RedisIP, port=RedisPort, db=RedisDB, password=RedisPassword)
                                try:
                                    if r1.ping():
                                        r1.hmset(mongoCName, resJson)
                                except Exception as e:
                                    print(e)
                        except Exception as e:
                            print(e)
                        try:
                            # print(resJson)
                            #
                            # print(resJson['timestamp'])
                            postsT.insert_one(resJson)
                            # print(2)
                        except pymongo.errors.DuplicateKeyError as e:
                            print(
                                '索引重复,此次插入失败,重复索引为{}'.format(
                                    resJson['timestamp']))
                        # print(3)
                        os.remove(unzipFile_path)
                        print('已删除:' + unzipFile_path)
                        break

                    except Exception as e:
                        print(e)
        conn.close()
        print("over")


if __name__ == '__main__':
    class MyThreadingTCPServer(socketserver.ThreadingTCPServer):
        """重写socketserver.ThreadingTCPServer"""
        # 服务停止后即刻释放端口,无需等待tcp连接断开
        allow_reuse_address = True
    dataStoreDir, dataToRecory_dir, mongoIP, mongoPort, mongoDbName, mongoCName, runPort, RedisIP, RedisPort, RedisDB, RedisPassword = initparse()
    print(
        dataStoreDir,
        dataToRecory_dir,
        mongoIP,
        mongoPort,
        mongoDbName,
        mongoCName,
        runPort)
    instance = MyThreadingTCPServer(('0.0.0.0', runPort), myserver)
    # poll_interval = 5
    instance.serve_forever()

求帮助,解决TCP服务端CPU占用过高问题_第1张图片这是运行几小时之后的cpu占用,突然变高的
求帮助,解决TCP服务端CPU占用过高问题_第2张图片
这是运行端口下的部分线程

你可能感兴趣的:(求帮助,解决TCP服务端CPU占用过高问题)