【Erlang】优化排查工具
%%%-------------------------------------------------------------------
%%% @author cnkizy
%%% @copyright (C) 2020
%%% @doc
%%% VM内部调优排查
%%% https://www.cnblogs.com/lulu/p/4149204.html
%%% @end
%%% Created : 2021/1/23-12-05
%%%-------------------------------------------------------------------
-module(analysis_lib).
-description("analysis_lib").
-author("cnkizy").
-vsn(1).
%% API
-export([fp_pid/1, fp_apply/3, pstack/1, etop/0, etop_mem/0, etop_stop/0, gc_all/0, eprof_all/1, eprof/2]).
-export([scheduler_usage/0, scheduler_usage/1]).
-export([scheduler_stat/0, scheduler_stat/1]).
-export([trace/1, trace/2, trace_stop/0]).
-export([proc_mem_all/1, proc_mem/1, proc_mem/2]).
%% -----------------------------------------------------------------
%% Function: fp_apply/3
%% Description: 执行一次MFA并进行过程分析,fprof 结果比较详细,能够输出热点调用路径
%% CNT表示总的函数调用次数，ACC表示Trace起始经历的时间，OWN是Trace中所有函数执行的时间和（不包含调用函数的执行时间
%% 时间单位:ms
%% Detail Reference: https://www.it610.com/article/519481.htm
%% -----------------------------------------------------------------
fp_apply(M, F, A) ->
fprof:start(),
fprof:apply(M, F, A),
fprof:stop(),
fprof:profile(),
fprof:analyse({dest, "profile.txt"}).
fp_pid(Pid) ->
fprof:start(),
fprof:trace([start, {procs, Pid}]),
fprof:stop(),
fprof:profile(),
fprof:analyse({dest, "profile.txt"}).
%% ================================== 进程栈 =======================================
%% -----------------------------------------------------------------
%% Function: pstack/3
%% Description: 类似于jstack，发现大量进程挂起，进程数过高，运行慢，hang住等问题用到
%% -----------------------------------------------------------------
pstack(Reg) when is_atom(Reg) ->
case whereis(Reg) of
undefined -> undefined;
Pid -> pstack(Pid)
end;
pstack(Pid) ->
io:format("~s~n", [element(2, process_info(Pid, backtrace))]).
%% -----------------------------------------------------------------
%% Function: etop/0
%% Description: 启动etop 每10秒输出一次 当前进程CPU占用排名
%% -----------------------------------------------------------------
etop() ->
spawn(fun() -> etop:start([{output, text}, {interval, 10}, {lines, 20}, {sort, reductions}]) end).
%% -----------------------------------------------------------------
%% Function: etop_mem/0
%% Description: 启动etop 每10秒输出一次 进程Mem占用排名
%% -----------------------------------------------------------------
etop_mem() ->
spawn(fun() -> etop:start([{output, text}, {interval, 10}, {lines, 20}, {sort, memory}]) end).
%% -----------------------------------------------------------------
%% Function: etop_stop/0
%% Description: 停止etop
%% -----------------------------------------------------------------
etop_stop() ->
etop:stop().
%% -----------------------------------------------------------------
%% Function: gc_all/0
%% Description: 对所有process做gc、进程内存过高时，来一发，看看是内存泄露还是gc不过来
%% -----------------------------------------------------------------
gc_all() ->
[erlang:garbage_collect(Pid) || Pid <- processes()].
%% -----------------------------------------------------------------
%% Function: eprof_all/1
%% Description:
%%
%% 对整个节点内所有进程执行eprof, eprof 对线上业务有一定影响,慎用!
%% 建议TimeoutSec<10s，且进程数< 1000，否则可能导致节点crash
%%
%% 结果:
%% 输出每个方法实际执行时间（不会累计方法内其他mod调用执行时间）
%% 只能得到mod - Fun 执行次数 执行耗时
%% -----------------------------------------------------------------
eprof_all(TimeoutSec) ->
eprof(processes() -- [whereis(eprof)], TimeoutSec).
eprof(Pids, TimeoutSec) ->
eprof:start(),
eprof:start_profiling(Pids),
timer:sleep(TimeoutSec),
eprof:stop_profiling(),
eprof:analyze(total),
eprof:stop().
%% -----------------------------------------------------------------
%% Function: scheduler_usage/0
%% Description: 统计下1s每个调度器CPU的实际利用率(因为有spin wait、调度工作, 可能usage 比top显示低很多)
%% -----------------------------------------------------------------
scheduler_usage() ->
scheduler_usage(1000).
scheduler_usage(RunMs) ->
erlang:system_flag(scheduler_wall_time, true),
Ts0 = lists:sort(erlang:statistics(scheduler_wall_time)),
timer:sleep(RunMs),
Ts1 = lists:sort(erlang:statistics(scheduler_wall_time)),
erlang:system_flag(scheduler_wall_time, false),
Cores = lists:map(
fun({{I, A0, T0}, {I, A1, T1}}) ->
{I, (A1 - A0) / (T1 - T0)}
end, lists:zip(Ts0, Ts1)),
{A, T} = lists:foldl(
fun({{_, A0, T0}, {_, A1, T1}}, {Ai, Ti}) ->
{Ai + (A1 - A0), Ti + (T1 - T0)}
end, {0, 0}, lists:zip(Ts0, Ts1)),
Total = A / T,
io:format("~p~n", [[{total, Total} | Cores]]).
%% -----------------------------------------------------------------
%% Function: scheduler_stat/0
%% Description: 统计下1s内调度进程数量(含义：第一个数字执行进程数量，第二个数字迁移进程数量)
%% -----------------------------------------------------------------
scheduler_stat() ->
scheduler_stat(1000).
scheduler_stat(RunMs) ->
erlang:system_flag(scheduling_statistics, enable),
Ts0 = erlang:system_info(total_scheduling_statistics),
timer:sleep(RunMs),
Ts1 = erlang:system_info(total_scheduling_statistics),
erlang:system_flag(scheduling_statistics, disable),
lists:map(fun scheduler_stat_/1, lists:zip(Ts0, Ts1)).
scheduler_stat_({{Key, In0, Out0}, {Key, In1, Out1}}) ->
{Key, In1 - In0, Out1 - Out0}.
%% -----------------------------------------------------------------
%% Function: trace/1
%% Description: 会把mod 每次调用详细MFA log 下来，args 太大就不好看了
%% trace Mod 所有方法的调用
%% -----------------------------------------------------------------
trace(Mod) ->
dbg:tracer(),
dbg:tpl(Mod, '_', []),
dbg:p(all, c).
%% -----------------------------------------------------------------
%% Function: trace/2
%% Description: trace Node上指定 Mod 所有方法的调用, 结果将输出到本地shell
%% -----------------------------------------------------------------
trace(Node, Mod) ->
dbg:tracer(),
dbg:n(Node),
dbg:tpl(Mod, '_', []),
dbg:p(all, c).
%% -----------------------------------------------------------------
%% Function: trace_stop/0
%% Description: 停止trace
%% -----------------------------------------------------------------
trace_stop() ->
dbg:stop_clear().
%% ================================== 内存高OOM 排查工具 =======================================
%% -----------------------------------------------------------------
%% Function: proc_mem_all/1
%% Description: etop 无法应对10w+ 进程节点, 下面代码就没问题了；找到可疑proc后通过pstack、message_queu_len 排查原因
%% -----------------------------------------------------------------
proc_mem_all(SizeLimitKb) ->
Procs = [{undefined, Pid} || Pid <- erlang:processes()],
proc_mem(Procs, SizeLimitKb).
%% -----------------------------------------------------------------
%% Function: proc_mem/1
%% Description: etop 无法应对10w+ 进程节点, 下面代码就没问题了；找到可疑proc后通过pstack、message_queu_len 排查原因
%% -----------------------------------------------------------------
proc_mem(SizeLimitKb) ->
Procs = [{Name, Pid} || {_, Name, Pid, _} <- release_handler_1:get_supervised_procs(),
is_process_alive(Pid)],
proc_mem(Procs, SizeLimitKb).
proc_mem(Procs, SizeLimitKb) ->
SizeLimit = SizeLimitKb * 1024,
{R, Total} = lists:foldl(fun({Name, Pid}, {Acc, TotalSize}) ->
case erlang:process_info(Pid, total_heap_size) of
{_, Size0} ->
Size = Size0 * 8,
case Size > SizeLimit of
true -> {[{Name, Pid, Size} | Acc], TotalSize + Size};
false -> {Acc, TotalSize}
end;
_ -> {Acc, TotalSize}
end
end, {[], 0}, Procs),
R1 = lists:keysort(3, R),
{Total, lists:reverse(R1)}.
分析单个函数时，fprof较为实用。
褚霸发过一遍fprof http://blog.yufeng.info/archives/tag/fprof 弄感兴趣的数据收集才有意义。