framework watchdog
问题背景
搞稳定性以来，知道framework有个watchdog，监听主要进程是否卡死。
如果卡住60秒就会杀死所在进程也就是system_server，上层重启。
搜索关键log “WATCHDOG KILLING SYSTEM PROCESS”即可。
那么它到底是如何判定系统卡住了，这里简单了来看一下watchdog的代码了解一下
watchdog
在源码里添加一些log，方便分析：
private void run() {
boolean waitedHalf = false;
while (true) {
List<HandlerChecker> blockedCheckers = Collections.emptyList();
String subject = "";
boolean allowRestart = true;
int debuggerWasConnected = 0;
boolean doWaitedHalfDump = false;
// The value of mWatchdogTimeoutMillis might change while we are executing the loop.
// We store the current value to use a consistent value for all handlers.
final long watchdogTimeoutMillis = mWatchdogTimeoutMillis;
final long checkIntervalMillis = watchdogTimeoutMillis / 2;
//watchdogTimeoutMillis 60000
//checkIntervalMillis 30000
Slog.i(TAG, "watchdog run watchdogTimeoutMillis: " + watchdogTimeoutMillis+" checkIntervalMillis: "+checkIntervalMillis);
final ArrayList<Integer> pids;
...
synchronized (mLock) {
long timeout = checkIntervalMillis;
//timeout 30000
Slog.d(TAG, "watchdog run timeout 111 : " + timeout);
// Make sure we (re)spin the checkers that have become idle within
// this wait-and-check interval
for (int i=0; i<mHandlerCheckers.size(); i++) {
HandlerCheckerAndTimeout hc = mHandlerCheckers.get(i);
// We pick the watchdog to apply every time we reschedule the checkers. The
// default timeout might have changed since the last run.
//scheduleCheckLocked---记录startTime
hc.checker().scheduleCheckLocked(hc.customTimeoutMillis()
.orElse(watchdogTimeoutMillis * Build.HW_TIMEOUT_MULTIPLIER));
}
if (debuggerWasConnected > 0) {
debuggerWasConnected--;
}
// NOTE: We use uptimeMillis() here because we do not want to increment the time we
// wait while asleep. If the device is asleep then the thing that we are waiting
// to timeout on is asleep as well and won't have a chance to run, causing a false
// positive on when to kill things.
long start = SystemClock.uptimeMillis();
//start 的是从系统启动开始以来的时间
Slog.i(TAG, "watchdog run start: " + start);
while (timeout > 0) {
if (Debug.isDebuggerConnected()) {
debuggerWasConnected = 2;
}
try {
//wait 30000毫秒
mLock.wait(timeout);
// Note: mHandlerCheckers and mMonitorChecker may have changed after waiting
} catch (InterruptedException e) {
Log.wtf(TAG, e);
}
if (Debug.isDebuggerConnected()) {
debuggerWasConnected = 2;
}
//30000 - 时间差
//如果程序完全不耗时，因为上面wait 30000，这里趋近于0
//但是代码运行总是需要时间的，所以timeout打印出来是-1、-2毫秒
timeout = checkIntervalMillis - (SystemClock.uptimeMillis() - start);
Slog.d(TAG, "watchdog run timeout 222 : " + timeout);
}
//获取状态，后面主要就靠这个状态判断是否要kill重启
final int waitState = evaluateCheckerCompletionLocked();
Slog.d(TAG, "watchdog run waitState : " + waitState);
//30000毫秒内，检查完所有进程，直接continue进行下一轮检测
if (waitState == COMPLETED) {
// The monitors have returned; reset
waitedHalf = false;
continue;
} else if (waitState == WAITING) {
// still waiting but within their configured intervals; back off and recheck
...
continue;
} else if (waitState == WAITED_HALF) {
//waitedHalf最开始是false，第一次等待超时，进入后改为true
//第一次超时会进入，第二次不会重复进
//配合下面的doWaitedHalfDump 实现第一次超时不重启，第二次重启
if (!waitedHalf) {
Slog.i(TAG, "WAITED_HALF");
waitedHalf = true;
// We've waited half, but we'd need to do the stack trace dump w/o the lock.
blockedCheckers = getCheckersWithStateLocked(WAITED_HALF);
subject = describeCheckersLocked(blockedCheckers);
pids = new ArrayList<>(mInterestingJavaPids);
doWaitedHalfDump = true;
} else {
continue;
}
} else {
//如果走到这里，说明要杀死pid了，系统即将重启
// something is overdue!
blockedCheckers = getCheckersWithStateLocked(OVERDUE);
subject = describeCheckersLocked(blockedCheckers);
allowRestart = mAllowRestart;
pids = new ArrayList<>(mInterestingJavaPids);
}
} // END synchronized (mLock)
// If we got here, that means that the system is most likely hung.
//
// First collect stack traces from all threads of the system process.
//
// Then, if we reached the full timeout, kill this process so that the system will
// restart. If we reached half of the timeout, just log some information and continue.
logWatchog(doWaitedHalfDump, subject, pids);
//doWaitedHalfDump 在第一次超时是赋值为true，保证log能打印，第二次才会真正的kill
if (doWaitedHalfDump) {
// We have waited for only half of the timeout, we continue to wait for the duration
// of the full timeout before killing the process.
continue;
}
IActivityController controller;
synchronized (mLock) {
controller = mController;
}
if (controller != null) {
Slog.i(TAG, "Reporting stuck state to activity controller");
try {
Binder.setDumpDisabled("Service dumps disabled due to hung system process.");
// 1 = keep waiting, -1 = kill system
int res = controller.systemNotResponding(subject);
if (res >= 0) {
Slog.i(TAG, "Activity controller requested to coninue to wait");
waitedHalf = false;
continue;
}
} catch (RemoteException e) {
}
}
//这个是判断是否连接了调试程序，断电时防止直接kill系统进程，正常不会走这里
// Only kill the process if the debugger is not attached.
if (Debug.isDebuggerConnected()) {
debuggerWasConnected = 2;
}
if (debuggerWasConnected >= 2) {
Slog.w(TAG, "Debugger connected: Watchdog is *not* killing the system process");
} else if (debuggerWasConnected > 0) {
Slog.w(TAG, "Debugger was connected: Watchdog is *not* killing the system process");
} else if (!allowRestart) {
Slog.w(TAG, "Restart not allowed: Watchdog is *not* killing the system process");
} else {
//这里就是kill系统进程的地方
Slog.w(TAG, "*** WATCHDOG KILLING SYSTEM PROCESS: " + subject);
WatchdogDiagnostics.diagnoseCheckers(blockedCheckers);
Slog.w(TAG, "*** GOODBYE!");
if (!Build.IS_USER && isCrashLoopFound()
&& !WatchdogProperties.should_ignore_fatal_count().orElse(false)) {
breakCrashLoop();
}
Process.killProcess(Process.myPid());
System.exit(10);
}
waitedHalf = false;
}
}
整个方法有2个地方需要注意
如何判断系统是否卡住了、waitState如何赋值
1、hc.checker().scheduleCheckLocked
public void scheduleCheckLocked(long handlerCheckerTimeoutMillis) {
mWaitMaxMillis = handlerCheckerTimeoutMillis;
if (mCompleted) {
// Safe to update monitors in queue, Handler is not in the middle of work
mMonitors.addAll(mMonitorQueue);
mMonitorQueue.clear();
}
//监视队列没有东西，或者队列正在轮询，直接完成
if ((mMonitors.size() == 0 && mHandler.getLooper().getQueue().isPolling())
|| (mPauseCount > 0)) {
// Don't schedule until after resume OR
// If the target looper has recently been polling, then
// there is no reason to enqueue our checker on it since that
// is as good as it not being deadlocked. This avoid having
// to do a context switch to check the thread. Note that we
// only do this if we have no monitors since those would need to
// be executed at this point.
mCompleted = true;
return;
}
//如果上次检查没有完成，不进行重复操作，直接return
if (!mCompleted) {
// we already have a check in flight, so no need
return;
}
//修改标志位，表示检查没有完成
mCompleted = false;
mCurrentMonitor = null;
//记录启动查询的时间
mStartTimeMillis = SystemClock.uptimeMillis();
//mWaitMaxMillis 60000
Slog.i(TAG, "scheduleCheckLocked mWaitMaxMillis: " + mWaitMaxMillis);
//mStartTimeMillis 从系统启动开始以来的时间
Slog.i(TAG, "scheduleCheckLocked mStartTimeMillis: " + mStartTimeMillis);
//发消息进行检查工作
mHandler.postAtFrontOfQueue(this);
}
mHandler.run
@Override
public void run() {
// Once we get here, we ensure that mMonitors does not change even if we call
// #addMonitorLocked because we first add the new monitors to mMonitorQueue and
// move them to mMonitors on the next schedule when mCompleted is true, at which
// point we have completed execution of this method.
final int size = mMonitors.size();
//循环调用所有监视器的monitor方法
for (int i = 0 ; i < size ; i++) {
synchronized (mLock) {
mCurrentMonitor = mMonitors.get(i);
}
Slog.i(TAG, "HandlerChecker mCurrentMonitor: " + mCurrentMonitor);
mCurrentMonitor.monitor();
}
//检查完mCompleted改为true
synchronized (mLock) {
mCompleted = true;
mCurrentMonitor = null;
}
}
//检测的log
从log可以看出，watchdog的run和mHandler.run不在同一个线程。
S022A1B 09-28 18:41:00.033 1132 1198 I Watchdog: watchdog run watchdogTimeoutMillis: 60000 checkIntervalMillis: 30000
S022A1D 09-28 18:41:00.034 1132 1198 D Watchdog: watchdog run timeout 111 : 30000
S022A1E 09-28 18:41:00.034 1132 1198 I Watchdog: scheduleCheckLocked mWaitMaxMillis: 60000
S022A1F 09-28 18:41:00.035 1132 1198 I Watchdog: scheduleCheckLocked mStartTimeMillis: 4496639
S022A20 09-28 18:41:00.036 1132 1198 I Watchdog: watchdog run start: 4496641
S022A21 09-28 18:41:00.036 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.Watchdog$BinderThreadMonitor@75334b9
S022A22 09-28 18:41:00.037 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.am.UnisocActivityManagerServiceImpl@7dfd5fd
S022A24 09-28 18:41:00.038 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.power.PowerManagerService@877361c
S022A25 09-28 18:41:00.038 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.wm.UnisocWindowManagerService@ede477e
S022A26 09-28 18:41:00.039 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.input.InputManagerService@2e6c225
S022A27 09-28 18:41:00.041 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.StorageManagerService@9061ffa
S022A28 09-28 18:41:00.043 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.media.MediaSessionService@efcfeeb
S022A29 09-28 18:41:00.043 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.media.MediaRouterService@c265448
S022A2A 09-28 18:41:00.044 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.media.projection.MediaProjectionManagerService@1b002e1
S022A2B 09-28 18:41:00.044 1132 1191 I Watchdog: HandlerChecker mCurrentMonitor: com.android.server.am.BatteryStatsService@3d7f806
S022FCD 09-28 18:41:30.037 1132 1198 D Watchdog: watchdog run timeout 222 : 0
S022FCE 09-28 18:41:30.037 1132 1198 D Watchdog: watchdog run waitState : 0
monitor里面啥也没有。只是获取一下同步锁，看看service有没有在运行其它任务。
如果长期运行其它任务，就无法获取到mLock。
frameworks\base\services\core\java\com\android\server\power\PowerManagerService.java
public void monitor() {
// Grab and release lock for watchdog monitor to detect deadlocks.
synchronized (mLock) {
}
}
2、evaluateCheckerCompletionLocked
private int evaluateCheckerCompletionLocked() {
int state = COMPLETED;
for (int i=0; i<mHandlerCheckers.size(); i++) {
HandlerChecker hc = mHandlerCheckers.get(i).checker();
state = Math.max(state, hc.getCompletionStateLocked());
}
return state;
}
public int getCompletionStateLocked() {
Slog.i(TAG, "getCompletionStateLocked mCompleted: " + mCompleted);
//上面的检测如果在30秒内完成，mCompleted就是true
if (mCompleted) {
return COMPLETED;
} else {
//如果没完成，则根据卡顿的时间来计算状态
//因为上面wait了30000毫秒，所以latency一定是大于30000毫秒的
long latency = SystemClock.uptimeMillis() - mStartTimeMillis;
Slog.i(TAG, "getCompletionStateLocked latency: " + latency);
if (latency < mWaitMaxMillis / 2) {
return WAITING;//这里理论上不会走进来
} else if (latency < mWaitMaxMillis) {
return WAITED_HALF;//第一次卡顿
}
}
return OVERDUE;//第二次卡顿
}