下载地址： wget --no-check-certificate https://raw.github.com/oldratlee/useful-scripts/release-2.x/bin/show-busy-java-threads
脚本一
#!/bin/bash
# @Function
# Find out the highest cpu consumed threads of java processes, and print the stack of these threads.
#
# @Usage
# $ ./show-busy-java-threads
#
# @online-doc https://github.com/oldratlee/useful-scripts/blob/dev-2.x/docs/java.md#-show-busy-java-threads
# @author Jerry Lee (oldratlee at gmail dot com)
# @author superhj1987 (superhj1987 at 126 dot com)
readonly PROG="`basename $0`"
readonly -a COMMAND_LINE=("$0" "$@")
# Get current user name via whoami command
# See https://www.lifewire.com/current-linux-user-whoami-command-3867579
# Because if run command by `sudo -u`, env var $USER is not rewritten/correct, just inherited from outside!
readonly USER="`whoami`"
################################################################################
# util functions
################################################################################
# NOTE: $'foo' is the escape sequence syntax of bash
readonly ec=$'\033' # escape char
readonly eend=$'\033[0m' # escape end
colorEcho() {
local color=$1
shift
# if stdout is console, turn on color output.
[ -t 1 ] && echo "$ec[1;${color}m$@$eend" || echo "$@"
}
colorPrint() {
local color=$1
shift
colorEcho "$color" "$@"
[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"
[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}
normalPrint() {
echo "$@"
[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"
[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}
redPrint() {
colorPrint 31 "$@"
}
greenPrint() {
colorPrint 32 "$@"
}
yellowPrint() {
colorPrint 33 "$@"
}
bluePrint() {
colorPrint 36 "$@"
}
die() {
redPrint "Error: $@" 1>&2
exit 1
}
logAndRun() {
echo "$@"
echo
"$@"
}
logAndCat() {
echo "$@"
echo
cat
}
usage() {
local -r exit_code="$1"
shift
[ -n "$exit_code" -a "$exit_code" != 0 ] && local -r out=/dev/stderr || local -r out=/dev/stdout
(( $# > 0 )) && { echo "$@"; echo; } > $out
> $out cat < find out the highest cpu consumed threads from
the specified java process.
default from all java process.
-c, --count set the thread count to show, default is 5.
-a, --append-file specifies the file to append output as log.
-S, --store-dir specifies the directory for storing
the intermediate files, and keep files.
default store intermediate files at tmp dir,
and auto remove after run. use this option to keep
files so as to review jstack/top/ps output later.
delay the delay between updates in seconds.
count the number of updates.
delay/count arguments imitates the style of
vmstat command.
jstack control:
-s, --jstack-path specifies the path of jstack command.
-F, --force set jstack to force a thread dump. use when jstack
does not respond (process is hung).
-m, --mix-native-frames set jstack to print both java and native frames
(mixed mode).
-l, --lock-info set jstack with long listing.
prints additional information about locks.
CPU usage calculation control:
-d, --top-delay specifies the delay between top samples.
default is 0.5 (second). get thread cpu percentage
during this delay interval.
more info see top -d option. eg: -d 1 (1 second).
-P, --use-ps use ps command to find busy thread(cpu usage)
instead of top command.
default use top command, because cpu usage of
ps command is expressed as the percentage of
time spent running during the *entire lifetime*
of a process, this is not ideal in general.
Miscellaneous:
-h, --help display this help and exit.
EOF
exit $exit_code
}
################################################################################
# Check os support
################################################################################
uname | grep '^Linux' -q || die "$PROG only support Linux, not support `uname` yet!"
################################################################################
# parse options
################################################################################
# NOTE: ARGS can not be declared as readonly!!
# readonly declaration make exit code of assignment to be always 0, aka. the exit code of `getopt` in subshell is discarded.
# tested on bash 4.2.46
ARGS=`getopt -n "$PROG" -a -o p:c:a:s:S:Pd:Fmlh -l count:,pid:,append-file:,jstack-path:,store-dir:,use-ps,top-delay:,force,mix-native-frames,lock-info,help -- "$@"`
[ $? -ne 0 ] && { echo; usage 1; }
eval set -- "${ARGS}"
while true; do
case "$1" in
-c|--count)
count="$2"
shift 2
;;
-p|--pid)
pid="$2"
shift 2
;;
-a|--append-file)
append_file="$2"
shift 2
;;
-s|--jstack-path)
jstack_path="$2"
shift 2
;;
-S|--store-dir)
store_dir="$2"
shift 2
;;
-P|--use-ps)
use_ps=true
shift
;;
-d|--top-delay)
top_delay="$2"
shift 2
;;
-F|--force)
force=-F
shift
;;
-m|--mix-native-frames)
mix_native_frames=-m
shift
;;
-l|--lock-info)
more_lock_info=-l
shift
;;
-h|--help)
usage
;;
--)
shift
break
;;
esac
done
count=${count:-5}
update_delay=${1:-0}
[ -z "$1" ] && update_count=1 || update_count=${2:-0}
(( update_count < 0 )) && update_count=0
top_delay=${top_delay:-0.5}
use_ps=${use_ps:-false}
# check the directory of append-file(-a) mode, create if not exsit.
if [ -n "$append_file" ]; then
if [ -e "$append_file" ]; then
[ -f "$append_file" ] || die "$append_file(specified by option -a, for storing run output files) exists but is not a file!"
[ -w "$append_file" ] || die "file $append_file(specified by option -a, for storing run output files) exists but is not writable!"
else
append_file_dir="$(dirname "$append_file")"
if [ -e "$append_file_dir" ]; then
[ -d "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not a directory!"
[ -w "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not writable!"
else
mkdir -p "$append_file_dir" || die "fail to create directory $append_file_dir(specified by option -a, for storing run output files)!"
fi
fi
fi
# check store directory(-S) mode, create directory if not exsit.
if [ -n "$store_dir" ]; then
if [ -e "$store_dir" ]; then
[ -d "$store_dir" ] || die "$store_dir(specified by option -S, for storing output files) exists but is not a directory!"
[ -w "$store_dir" ] || die "directory $store_dir(specified by option -S, for storing output files) exists but is not writable!"
else
mkdir -p "$store_dir" || die "fail to create directory $store_dir(specified by option -S, for storing output files)!"
fi
fi
################################################################################
# check the existence of jstack command
################################################################################
if [ -n "$jstack_path" ]; then
[ -f "$jstack_path" ] || die "$jstack_path is NOT found!"
[ -x "$jstack_path" ] || die "$jstack_path is NOT executalbe!"
elif which jstack &> /dev/null; then
jstack_path="`which jstack`"
else
[ -n "$JAVA_HOME" ] || die "jstack not found on PATH and No JAVA_HOME setting! Use -s option set jstack path manually."
[ -f "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) file does NOT exists! Use -s option set jstack path manually."
[ -x "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) is NOT executalbe! Use -s option set jstack path manually."
jstack_path="$JAVA_HOME/bin/jstack"
fi
################################################################################
# biz logic
################################################################################
readonly run_timestamp="`date "+%Y-%m-%d_%H:%M:%S.%N"`"
readonly uuid="${PROG}_${run_timestamp}_${RANDOM}_$$"
readonly tmp_store_dir="/tmp/${uuid}"
if [ -n "$store_dir" ]; then
readonly store_file_prefix="$store_dir/${run_timestamp}_"
else
readonly store_file_prefix="$tmp_store_dir/${run_timestamp}_"
fi
mkdir -p "$tmp_store_dir"
cleanupWhenExit() {
rm -rf "$tmp_store_dir" &> /dev/null
}
trap "cleanupWhenExit" EXIT
headInfo() {
colorEcho "0;34;42" ================================================================================
echo "$(date "+%Y-%m-%d %H:%M:%S.%N") [$(( i + 1 ))/$update_count]: ${COMMAND_LINE[@]}"
colorEcho "0;34;42" ================================================================================
echo
}
if [ -n "${pid}" ]; then
readonly ps_process_select_options="-p $pid"
else
readonly ps_process_select_options="-C java -C jsvc"
fi
# output field: pid, thread id(lwp), pcpu, user
# order by pcpu(percentage of cpu usage)
findBusyJavaThreadsByPs() {
# 1. sort by %cpu by ps option `--sort -pcpu`
# 2. use wide output(unlimited width) by ps option `-ww`
# avoid trunk user column to username_fo+ or $uid alike
local -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,pcpu,user --sort -pcpu --no-headers)
local -r ps_out="$("${ps_cmd_line[@]}")"
if [ -n "$store_dir" ]; then
echo "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"
fi
echo "$ps_out" | head -n "${count}"
}
# top with output field: thread id, %cpu
__top_threadId_cpu() {
# 1. sort by %cpu by top option `-o %CPU`
# unfortunately, top version 3.2 does not support -o option(supports from top version 3.3+),
# use
# HOME="$tmp_store_dir" top -H -b -n 1
# combined
# sort
# instead of
# HOME="$tmp_store_dir" top -H -b -n 1 -o '%CPU'
# 2. change HOME env var when run top,
# so as to prevent top command output format being change by .toprc user config file unexpectedly
# 3. use option `-d 0.5`(update interval 0.5 second) and `-n 2`(update 2 times),
# and use second time update data to get cpu percentage of thread in 0.5 second interval
# 4. top v3.3, there is 1 black line between 2 update;
# but top v3.2, there is 2 blank lines between 2 update!
local -a top_cmd_line=(top -H -b -d $top_delay -n 2)
local -r top_out=$(HOME="$tmp_store_dir" "${top_cmd_line[@]}")
if [ -n "$store_dir" ]; then
echo "$top_out" | logAndCat "${top_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_top"
fi
echo "$top_out" |
awk 'BEGIN { blockIndex = 0; currentLineHasText = 0; prevLineHasText = 0; } {
currentLineHasText = ($0 != "")
if (prevLineHasText && !currentLineHasText)
blockIndex++ # from text line to empty line, increase block index
if (blockIndex == 3 && ($NF == "java" || $NF == "jsvc")) # $NF(last field) is command field
# only print 4th text block(blockIndex == 3), aka. process info of second top update
print $1 " " $9 # $1 is thread id field, $9 is %cpu field
prevLineHasText = currentLineHasText # update prevLineHasText
}' | sort -k2,2nr
}
__complete_pid_user_by_ps() {
# ps output field: pid, thread id(lwp), user
local -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,user --no-headers)
local -r ps_out="$("${ps_cmd_line[@]}")"
if [ -n "$store_dir" ]; then
echo "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"
fi
local idx=0
local -a line
while IFS=" " read -a line ; do
(( idx < count )) || break
local threadId="${line[0]}"
local pcpu="${line[1]}"
# output field: pid, threadId, pcpu, user
local output_fields="$( echo "$ps_out" |
awk -v "threadId=$threadId" -v "pcpu=$pcpu" '$2==threadId {
printf "%s %s %s %s\n", $1, threadId, pcpu, $3; exit
}' )"
if [ -n "$output_fields" ]; then
(( idx++ ))
echo "$output_fields"
fi
done
}
# output format is same as function findBusyJavaThreadsByPs
findBusyJavaThreadsByTop() {
__top_threadId_cpu | __complete_pid_user_by_ps
}
printStackOfThreads() {
local -a line
local idx=0
while IFS=" " read -a line ; do
local pid="${line[0]}"
local threadId="${line[1]}"
local threadId0x="0x`printf %x ${threadId}`"
local pcpu="${line[2]}"
local user="${line[3]}"
(( idx++ ))
local jstackFile="${store_file_prefix}$(( i + 1 ))_jstack_${pid}"
[ -f "${jstackFile}" ] || {
local -a jstack_cmd_line=( "$jstack_path" ${force} $mix_native_frames $more_lock_info ${pid} )
if [ "${user}" == "${USER}" ]; then
# run without sudo, when java process user is current user
logAndRun "${jstack_cmd_line[@]}" > ${jstackFile}
elif [ $UID == 0 ]; then
# if java process user is not current user, must run jstack with sudo
logAndRun sudo -u "${user}" "${jstack_cmd_line[@]}" > ${jstackFile}
else
# current user is not root user, so can not run with sudo; print error message and rerun suggestion
redPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."
redPrint "User of java process($user) is not current user($USER), need sudo to rerun:"
yellowPrint " sudo ${COMMAND_LINE[@]}"
normalPrint
continue
fi || {
redPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."
normalPrint
rm "${jstackFile}" &> /dev/null
continue
}
}
bluePrint "[$idx] Busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user}):"
if [ -n "$mix_native_frames" ]; then
local sed_script="/--------------- $threadId ---------------/,/^---------------/ {
/--------------- $threadId ---------------/b # skip first separator line
/^---------------/d # delete second separator line
p
}"
elif [ -n "$force" ]; then
local sed_script="/^Thread ${threadId}:/,/^$/ {
/^$/d; p # delete end separator line
}"
else
local sed_script="/ nid=${threadId0x} /,/^$/ {
/^$/d; p # delete end separator line
}"
fi
{
sed "$sed_script" -n ${jstackFile}
echo
} | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"}
done
}
################################################################################
# Main
################################################################################
main() {
local i
# if update_count <= 0, infinite loop till user interrupted (eg: CTRL+C)
for (( i = 0; update_count <= 0 || i < update_count; ++i )); do
(( i > 0 )) && sleep "$update_delay"
[ -n "$append_file" -o -n "$store_dir" ] && headInfo | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"} > /dev/null
(( update_count != 1 )) && headInfo
if $use_ps; then
findBusyJavaThreadsByPs
else
findBusyJavaThreadsByTop
fi | printStackOfThreads
done
}
main
从所有运行的Java进程中找出最消耗CPU的线程（缺省5个），打印出其线程栈
缺省会自动从所有的Java进程中找出最消耗CPU的线程，这样用更方便
当然你可以手动指定要分析的Java进程Id，以保证只会显示你关心的那个Java进程的信息
show-busy-java-threads -p <指定的Java进程Id>
show-busy-java-threads -c <要显示的线程栈数>脚本二
下载地址：curl -O https://arthas.gitee.io/arthas-boot.jar
阿里开源的arthas现在已经几乎包揽了我们线上排查问题的工作，提供了一个很完整的工具集。在这个场景中，也只需要一个thread -n命令即可
要注意的是，arthas的cpu占比，和前面两种cpu占比统计方式不同。前面两种针对的是Java进程启动开始到现在的cpu占比情况，arthas这种是一段采样间隔内，当前JVM里各个线程所占用的cpu时间占总cpu时间的百分比。
具体见官网：https://alibaba.github.io/arthas/thread.html
转载地址 ：DevOps技术栈