JMX客户端及对commons-pool的监控
生产线上，我们的服务基本都是在linux环境下部署的，当高并发的时候，机器的负载是比较高的，所以我们只能在linux机器上执行一些简单的命令行工具，如jps，jstack，jinfo等，像重量级的jconsole，jvisualvm这些首先需要桌面环境才能观看，服务器肯定是没有开启X client的，所以只能通过jmx在远程客户端观看，但这种情况下，势必会对本机造成一些不必要的资源占用，如网络带宽等。但是想mbean这些对象的监控目前又只能通过jconsole，jvisualvm这些jmx客户端访问，想要在本机以命令行的方式查看还是比较困难的。本文的目的就是介绍下怎么在linux服务器下以命令下的方式进行jmx监控。
1、利用jmx连接本机java进程
直接看代码，代码写的有点乱，需要优化。
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import javax.management.MBeanAttributeInfo;
import javax.management.MBeanInfo;
import javax.management.MBeanServerConnection;
import javax.management.ObjectName;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
import com.sun.tools.attach.VirtualMachine;
import com.sun.tools.attach.VirtualMachineDescriptor;
public class JmxC {
public static void main(String[] args) {
if(args.length>0){
if(args[0].equals("-l")){
JmxC c = new JmxC();
c.m1();
}else if(args[0].equals("-p")){
int id = Integer.valueOf(args[1]);
System.out.println("pid:"+id);
JmxC c = new JmxC();
if(args.length > 2 && args[2].equals("-n")){
String name = args[3];
System.out.println("args name:"+name);
if(args.length > 4 && args[4].equals("-a")){
String atts = args[5];
System.out.println("args atts:"+name);
c.processMbean(c.getConnectStrById(id),name,atts);
}else{
c.processMbean(c.getConnectStrById(id),name,null);
}
}else{
c.processMbean(c.getConnectStrById(id));
}
}
}
}
public void processMbean(String connectorAddress,String name,String att){
if(null == connectorAddress) return;
JMXConnector connector = null;
try {
System.out.println("conn:" + connectorAddress);
JMXServiceURL url = new JMXServiceURL(connectorAddress);
connector = JMXConnectorFactory.connect(url);
MBeanServerConnection mbeanConn = connector.getMBeanServerConnection();
Set beanSet = mbeanConn.queryNames(null, null);
System.out.println("beanSet num:" + beanSet.size());
ObjectName n = new ObjectName(name);
MBeanInfo info = mbeanConn.getMBeanInfo(n);
if(null != info){
if(null != att && !att.isEmpty()){
String[] as = att.split(",");
List al = new LinkedList();
System.out.print(" time "+"\t");
for(String a:as){
if(null != a && !a.isEmpty()){
al.add(a);
System.out.print(a+"\t");
}
}
System.out.println();
SimpleDateFormat dateformat1=new SimpleDateFormat("HH:mm:ss");
while(true){
System.out.print(dateformat1.format(new Date())+"\t");
for(String a:al){
Object value = null;
if(!a.contains("-")){
value = mbeanConn.getAttribute(n, a);
}else{
String[] at= a.split("-");
value = (long)mbeanConn.getAttribute(n, at[0]) - (long)mbeanConn.getAttribute(n, at[1]);
}
System.out.print(value+"\t");
}
System.out.println();
Thread.sleep(1000);
}
}else{
MBeanAttributeInfo[] atts= info.getAttributes();
for(MBeanAttributeInfo attr : atts){
Object value = mbeanConn.getAttribute(n, attr.getName());
System.out.println(attr.getName()+"->"+value);
}
}
}else{
System.err.println("info is null");
}
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
if (connector != null) connector.close();
// break;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// ...
}
public void processMbean(String connectorAddress){
if(null == connectorAddress) return;
JMXConnector connector = null;
try {
System.out.println("conn:" + connectorAddress);
JMXServiceURL url = new JMXServiceURL(connectorAddress);
connector = JMXConnectorFactory.connect(url);
MBeanServerConnection mbeanConn = connector.getMBeanServerConnection();
Set beanSet = mbeanConn.queryNames(null, null);
System.out.println("beanSet num:" + beanSet.size());
for(ObjectName name: beanSet){
if(name.getDomain().equals("org.apache.commons.pool2")){
System.out.println("name:"+name);
//ObjectInstance instance = mbeanConn.getObjectInstance(name);
MBeanInfo info = mbeanConn.getMBeanInfo(name);
MBeanAttributeInfo[] atts= info.getAttributes();
for(MBeanAttributeInfo attr : atts){
Object value = mbeanConn.getAttribute(name, attr.getName());
System.out.println(attr.getName()+"->"+value);
}
System.out.println();
System.out.println();
System.out.println();
}
}
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
if (connector != null) connector.close();
// break;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// ...
}
public String getConnectStrById(int pid){
List vms = VirtualMachine.list();
int i = 0;
for (VirtualMachineDescriptor desc : vms) {
if(!desc.id().equals(""+pid)){
continue;
}
VirtualMachine vm;
try {
System.out.println("desc:" + desc);
System.out.println("process id:" + desc.id());
vm = VirtualMachine.attach(desc);
} catch (Exception e) {
e.printStackTrace();
continue;
}
try {
Properties props = vm.getAgentProperties();
System.out.println("args:"+props.get("sun.jvm.args"));
for (Map.Entry entry : props.entrySet()) {
// System.out.println(entry.getKey() + "->" +
// entry.getValue());
}
String connectorAddress = props.getProperty("com.sun.management.jmxremote.localConnectorAddress");
if (connectorAddress == null) {
System.out.println("connectorAddress is null,and continue search");
props = vm.getSystemProperties();
String home = props.getProperty("java.home");
// Normally in ${java.home}/jre/lib/management-agent.jar but
// might
// be in ${java.home}/lib in build environments.
String agent = home + File.separator + "jre" + File.separator + "lib" + File.separator + "management-agent.jar";
File f = new File(agent);
if (!f.exists()) {
agent = home + File.separator + "lib" + File.separator + "management-agent.jar";
f = new File(agent);
if (!f.exists()) {
throw new IOException("Management agent not found");
}
}
agent = f.getCanonicalPath();
vm.loadAgent(agent, "com.sun.management.jmxremote");
props = vm.getAgentProperties();
connectorAddress = props.getProperty("com.sun.management.jmxremote.localConnectorAddress");
if (connectorAddress == null) {
System.out.println("connectorAddress is null");
}
}
return connectorAddress;
} catch (Exception e) {
e.printStackTrace();
}
}
System.out.println("return null");
return null;
}
public void m1() {
List vms = VirtualMachine.list();
int i = 0;
for (VirtualMachineDescriptor desc : vms) {
System.out.println();
System.out.println();
System.out.println("------" + i++ + "----------------");
VirtualMachine vm;
try {
System.out.println("desc:" + desc);
System.out.println("process id:" + desc.id());
vm = VirtualMachine.attach(desc);
} catch (Exception e) {
e.printStackTrace();
continue;
}
JMXConnector connector = null;
try {
Properties props = vm.getAgentProperties();
System.out.println(props.get("sun.jvm.args"));
for (Map.Entry entry : props.entrySet()) {
// System.out.println(entry.getKey() + "->" +
// entry.getValue());
}
String connectorAddress = props.getProperty("com.sun.management.jmxremote.localConnectorAddress");
if (connectorAddress == null) {
System.out.println("connectorAddress is null,and continue search");
props = vm.getSystemProperties();
String home = props.getProperty("java.home");
// Normally in ${java.home}/jre/lib/management-agent.jar but
// might
// be in ${java.home}/lib in build environments.
String agent = home + File.separator + "jre" + File.separator + "lib" + File.separator + "management-agent.jar";
File f = new File(agent);
if (!f.exists()) {
agent = home + File.separator + "lib" + File.separator + "management-agent.jar";
f = new File(agent);
if (!f.exists()) {
throw new IOException("Management agent not found");
}
}
agent = f.getCanonicalPath();
vm.loadAgent(agent, "com.sun.management.jmxremote");
props = vm.getAgentProperties();
connectorAddress = props.getProperty("com.sun.management.jmxremote.localConnectorAddress");
if (connectorAddress == null) {
System.out.println("connectorAddress is null");
continue;
}
}
System.out.println("conn:" + connectorAddress);
JMXServiceURL url = new JMXServiceURL(connectorAddress);
connector = JMXConnectorFactory.connect(url);
MBeanServerConnection mbeanConn = connector.getMBeanServerConnection();
Set beanSet = mbeanConn.queryNames(null, null);
System.out.println("beanSet num:" + beanSet.size());
// ...
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (connector != null) connector.close();
// break;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
}
javac -cp $JAVA_HOME/lib/tools.jar JmxC.java
执行：
1、列车本机的java进程信息
java -cp .:$JAVA_HOME/lib/tools.jar JmxC -l
2、根据本机pid进行监控
java -cp .:$JAVA_HOME/lib/tools.jar JmxC -p 6007
3、根据进程pid和mbena名字监控对象
java -cp .:$JAVA_HOME/lib/tools.jar JmxC -p 6007 -n org.apache.commons.pool2:type=GenericObjectPool,name=pool3
4、根据进程pid和mbena名字和指定的字段名字监控
java -cp .:$JAVA_HOME/lib/tools.jar JmxC -p 5986 -n org.apache.commons.pool2:type=GenericObjectPool,name=pool3 -a NumActive,NumIdle,NumWaiters,BorrowedCount,ReturnedCount,BorrowedCount-ReturnedCount,CreatedCount,DestroyedCount,DestroyedByEvictorCount,MeanActiveTimeMillis,MeanIdleTimeMillis,MaxBorrowWaitTimeMillis,MeanBorrowWaitTimeMillis
2、commons-pool的mbean监控分析
commons-pool的GenerateObjectPool自带实现了一个用于监控池子状态的mbean，完整属性如下：
11:31 [[email protected]]$ sudo java -cp .:$JAVA_HOME/lib/tools.jar JmxC -p 6007 -n org.apache.commons.pool2:type=GenericObjectPool,name=pool3
pid:6007
args name:org.apache.commons.pool2:type=GenericObjectPool,name=pool3
desc:sun.tools.attach.LinuxAttachProvider@3d62b333: 6007 com.youku.java.navi.boot.NaviMain
process id:6007
args:-Xmx1024m -Xms1024m -XX:NewSize=384m -Xss512k -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -DJAVA_NAVI_HOME=/opt/yks/JavaNavi -Dfile.encoding=UTF-8 -Dlog4j.configuration=log4j.configuration -Dfile.log.path=/opt/yks/JavaNavi/logs/8084/8084 -Dlisten.port=8084
conn:service:jmx:rmi://127.0.0.1/stub/rO0ABXNyAC5qYXZheC5tYW5hZ2VtZW50LnJlbW90ZS5ybWkuUk1JU2VydmVySW1wbF9TdHViAAAAAAAAAAICAAB4cgAaamF2YS5ybWkuc2VydmVyLlJlbW90ZVN0dWLp/tzJi+FlGgIAAHhyABxqYXZhLnJtaS5zZXJ2ZXIuUmVtb3RlT2JqZWN002G0kQxhMx4DAAB4cHc4AAtVbmljYXN0UmVmMgAADTEwLjEwMy4yOS4xODYAAMj1VqDSGbCiEt0X5viNAAABVGz3DF2AAQB4
beanSet num:24
Closed->false
Lifo->true
MinEvictableIdleTimeMillis->60000
NumTestsPerEvictionRun->3
TestOnCreate->false
TestOnBorrow->false
TestOnReturn->false
TestWhileIdle->true
MaxWaitMillis->100
MaxIdle->200
MinIdle->0
TimeBetweenEvictionRunsMillis->30000
BlockWhenExhausted->true
MaxTotal->200
RemoveAbandonedOnMaintenance->false
AbandonedConfig->false
LogAbandoned->false
RemoveAbandonedOnBorrow->false
RemoveAbandonedTimeout->2147483647
NumActive->0
NumIdle->15
NumWaiters->0
FactoryType->com.youku.java.navi.fw.datasource.DefaultNaviDataSource$NaviPoolableObjectFactory
BorrowedCount->258650706
ReturnedCount->258649202
CreatedCount->5350
DestroyedCount->5335
DestroyedByEvictorCount->3827
DestroyedByBorrowValidationCount->0
MeanActiveTimeMillis->0
MeanIdleTimeMillis->8
MeanBorrowWaitTimeMillis->0
MaxBorrowWaitTimeMillis->0
CreationStackTrace->java.lang.Exception
at org.apache.commons.pool2.impl.BaseGenericObjectPool.(BaseGenericObjectPool.java:139)
at org.apache.commons.pool2.impl.GenericObjectPool.(GenericObjectPool.java:107)
at com.youku.java.navi.fw.datasource.DefaultNaviDataSource.initConnPool(DefaultNaviDataSource.java:58)
at com.youku.java.navi.fw.datasource.DefaultNaviDataSource.afterPropertiesSet(DefaultNaviDataSource.java:145)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1514)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1452)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:519)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:456)
at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:294)
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:225)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:291)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:193)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:609)
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:469)
at com.youku.java.navi.module.NaviClassPathXmlApplicationContext.refresh(NaviClassPathXmlApplicationContext.java:40)
at com.youku.java.navi.module.NaviModuleContext.initModule(NaviModuleContext.java:46)
at com.youku.java.navi.module.NaviModuleContextFactory$CheckModuleVersion.run(NaviModuleContextFactory.java:78)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:724) 关键的属性如下：
正常运行中：
停止接收客户请求1
停止接收客户请求2
接收客户请求
其中重点关注项：
NumActive
,NumIdle
,NumWaiters
,BorrowedCount
,ReturnedCount
,BorrowedCount-ReturnedCount 看上图比较大，是因为大部分是被destory过（连接失败或者操作失败等导致）的，所以没有记到ReturnedCount中
,CreatedCount
,DestroyedCount
,DestroyedByEvictorCount DestroyedCount - DestroyedByEvictorCount 即为由于连接失败等原因destory的
,MeanActiveTimeMillis
,MeanIdleTimeMillis
,MaxBorrowWaitTimeMillis
,MeanBorrowWaitTimeMillis
mean是平均值的意思
3、根据进程pid和mbena名字