#coding=utf-8
import threading
import subprocess
import sys
from queue import Queue
import re
import time
__author__='ao.deng'
'''
该工具主要用于在线分析频繁打印日志问题
主要的衡量指标是一秒钟和一分钟相同tag打印的次数
'''
ADB_STDOUT_READ_TIMEOUT = 0.2
# Minimum number of seconds between displaying status updates.
MIN_TIME_BETWEEN_STATUS_UPDATES = 1
class FileReaderThread(threading.Thread):
"""Reads data from a file/pipe on a worker thread.
Use the standard threading. Thread object API to start and interact with the
thread (start(), join(), etc.).
"""
def __init__(self, file_object, output_queue, text_file, chunk_size=-1):
"""Initializes a FileReaderThread.
Args:
file_object: The file or pipe to read from.
output_queue: A Queue.Queue object that will receive the data
text_file: If True, the file will be read one line at a time, and
chunk_size will be ignored. If False, line breaks are ignored and
chunk_size must be set to a positive integer.
chunk_size: When processing a non-text file (text_file = False),
chunk_size is the amount of data to copy into the queue with each
read operation. For text files, this parameter is ignored.
"""
threading.Thread.__init__(self)
self._file_object = file_object
self._output_queue = output_queue
self._text_file = text_file
self._chunk_size = chunk_size
assert text_file or chunk_size > 0
def run(self):
"""Overrides Thread's run() function.
Returns when an EOF is encountered.
"""
if self._text_file:
# Read a text file one line at a time.
for line in self._file_object:
self._output_queue.put(line)
else:
# Read binary or text data until we get to EOF.
while True:
chunk = self._file_object.read(self._chunk_size)
if not chunk:
break
self._output_queue.put(chunk)
def set_chunk_size(self, chunk_size):
"""Change the read chunk size.
This function can only be called if the FileReaderThread object was
created with an initial chunk_size > 0.
Args:
chunk_size: the new chunk size for this file. Must be > 0.
"""
# The chunk size can be changed asynchronously while a file is being read
# in a worker thread. However, type of file can not be changed after the
# the FileReaderThread has been created. These asserts verify that we are
# only changing the chunk size, and not the type of file.
assert not self._text_file
assert chunk_size > 0
self._chunk_size = chunk_size
class OnlineCheckLogLines():
def __init__(self):
self._adb = None
self.tagMap = {}
self.allLogLines=0
self.logExit=[]
self._log_args =["adb","shell","logcat","-b","all"]
def start(self):
self._adb = do_popen(self._log_args)
def _collect_log_data(self):
# Read the output from ADB in a worker thread. This allows us to monitor
# the progress of ADB and bail if ADB becomes unresponsive for any reason.
# Limit the stdout_queue to 128 entries because we will initially be reading
# one byte at a time. When the queue fills up, the reader thread will
# block until there is room in the queue. Once we start downloading the
# trace data, we will switch to reading data in larger chunks, and 128
# entries should be plenty for that purpose.
stdout_queue = Queue(maxsize=128)
stderr_queue = Queue()
# Use a chunk_size of 1 for stdout so we can display the output to
# the user without waiting for a full line to be sent.
stdout_thread = FileReaderThread(self._adb.stdout, stdout_queue,
text_file=True)
stderr_thread = FileReaderThread(self._adb.stderr, stderr_queue,
text_file=True)
stdout_thread.start()
stderr_thread.start()
# Holds the trace data returned by ADB.
log_data = []
last_status_update_time = time.time()
while (stdout_thread.isAlive() or stderr_thread.isAlive() or
not stdout_queue.empty() or not stderr_queue.empty()):
last_status_update_time = self.status_update(last_status_update_time)
while not stderr_queue.empty():
# Pass along errors from adb.
line = stderr_queue.get()
line = bytes.decode(line, errors="ignore")
sys.stderr.write(line)
# Read stdout from adb. The loop exits if we don't get any data for
# ADB_STDOUT_READ_TIMEOUT seconds.
while True:
try:
chunk = stdout_queue.get(True, ADB_STDOUT_READ_TIMEOUT)
except Exception as e:
# Didn't get any data, so exit the loop to check that ADB is still
# alive and print anything sent to stderr.
break
# Save, but don't print, the trace data.
chunk = bytes.decode(chunk,errors="ignore")
#sys.stdout.write(chunk)
self.extractTag(chunk)
last_status_update_time = self.status_update(last_status_update_time)
#log_data.append(chunk)
# The threads should already have stopped, so this is just for cleanup.
stdout_thread.join()
stderr_thread.join()
self._adb.stdout.close()
self._adb.stderr.close()
# The adb process should be done since it's io pipes are closed. Call
# poll() to set the returncode.
self._adb.poll()
if self._adb.returncode != 0:
print(">> {} {}".format(sys.stderr, ('The command "%s" returned error code %d.' %
(' '.join(self._log_args), self._adb.returncode))))
sys.exit(1)
return log_data
def extractTag(self,line):
pattern = "\d{2}-\d{2} (\d{2}:\d{2}:\d{2}.\d+)\s+\d+\s+\d+\s+\w (.*?)(\s+)?:"
timeAndTagList = re.findall(pattern, line)
if len(timeAndTagList)==0:
return
#print(timeAndTagList)
time = timeAndTagList[0][0]
tag = timeAndTagList[0][1]
self.allLogLines+=len(timeAndTagList)
if self.tagMap.keys().__contains__(tag):
self.tagMap[tag].append(time)
else:
timeList = [time]
self.tagMap[tag] = timeList
def countOneSecondOfData(self,timeList):
secondMap = {}
for time in timeList:
second = time.split(".")[0]
if secondMap.keys().__contains__(second):
secondMap[second] += 1
else:
secondMap[second] = 1
return secondMap
def status_update(self, last_update_time):
current_time = time.time()
if (current_time - last_update_time) >= MIN_TIME_BETWEEN_STATUS_UPDATES:
# Gathering a trace may take a while. Keep printing something so users
# don't think the script has hung.
for tag, timeList in self.tagMap.items():
secondMap = self.countOneSecondOfData(timeList)
for second, count in secondMap.items():
if count > 100:
line_str ="tag={}, lines={}, all log %={:.2f}, time={}, count={},".format(tag,len(timeList),len(timeList)/self.allLogLines, second, count)
line_str_key = "tag={}, time={}, count={},".format(tag,second, count)
if not self.logExit.__contains__(line_str_key):
print(line_str)
self.logExit.append(line_str_key)
# sys.stdout.write("\r{0}".format(11))
# sys.stdout.flush()
return current_time
return last_update_time
def do_popen(args):
print(args)
try:
adb = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as error:
print (">> {} {}".format(sys.stderr, (
'The command "%s" failed with the following error:' %
' '.join(args))))
print (">> {} {}".format(sys.stderr, ' ', error))
sys.exit(1)
return adb
OnlineCheckLogLinesWrapper = OnlineCheckLogLines()
OnlineCheckLogLinesWrapper.start()
OnlineCheckLogLinesWrapper._collect_log_data()
