cat监控小结
1、分布式logview的日志树串联实现，目前试验过两种，一种是基于dubbo应用的rpc调用，一种是基于http请求的rest服务调用。首先说下message tree的实现，追踪跨服务的消息时，通过根消息id和父级消息id及子消息id三个属性进行消息串联，组成消息树。关键点在tree的三个id的获得和传递。
这里有两点，第一是CAT消息树生成原理：
我们需要实现Cat的Context上下文，然后通过Cat.logRemoteCallClient(context)生成包含节点数据的上下文对象（方法中通过创建消息树对象来获取各节点的消息id，填充给上下文），远程服务端接收到这个context，使用Cat.logRemoteCallServer(context)方法，读取各节点消息id，组建消息树。
第二是消息应如何传递：
dubbo应用的rpc调用方式：调用过程要传递的rpc上下文，其中包含调用信息、参数以及状态信息等，可以把消息id信息放到RpcContext中，然后通过调用Invocation对象的invoke方法，将消息传递至服务端。最后，通过dubbo的spi拓展机制，实现com.alibaba.dubbo.rpc.Filter，用来获取rpcContext的内容。
rest风格的http请求方式：调用时，在服务请求方把消息id信息放到 Http-Header 中，在服务提供方，用filter拦截，并获得 http-header 中的消息id，这样通过埋点，串联起消息树。
1).以下是dubbo方式部分代码实现（首先要清楚dubbo配置文件的相关配置，CAT配置文件的配置等）：
public class DubboCatFilter implements Filter {
private static final ThreadLocal CAT_CONTEXT = new ThreadLocal();
@Override
public Result invoke(Invoker invoker, Invocation invocation) throws RpcException {
URL url = invoker.getUrl();
String sideKey = url.getParameter(Constants.SIDE_KEY);
String loggerName = invoker.getInterface().getSimpleName() + "." + invocation.getMethodName();
String type = "PigeonCall";
if (Constants.PROVIDER_SIDE.equals(sideKey)) {
type = "PigeonService";
}
Transaction t = Cat.newTransaction(type, loggerName);
Result result = null;
try {
Cat.Context context = getContext();
if (Constants.CONSUMER_SIDE.equals(sideKey)) {
createConsumerCross(url, t);
Cat.logRemoteCallClient(context);
} else {
createProviderCross(url, t);
Cat.logRemoteCallServer(context);
}
setAttachment(context);
result = invoker.invoke(invocation);
if (result.hasException()) {
//给调用接口出现异常进行打点
Throwable throwable = result.getException();
Event event = null;
if (RpcException.class == throwable.getClass()) {
Throwable caseBy = throwable.getCause();
if (caseBy != null && caseBy.getClass() == TimeoutException.class) {
event = Cat.newEvent("DUBBO_TIMEOUT_ERROR", loggerName);
} else {
event = Cat.newEvent("DUBBO_REMOTING_ERROR", loggerName);
}
} else if (RemotingException.class.isAssignableFrom(throwable.getClass())) {
event = Cat.newEvent("DUBBO_REMOTING_ERROR", loggerName);
}else{
event = Cat.newEvent("DUBBO_BIZ_ERROR", loggerName);
}
event.setStatus(result.getException());
completeEvent(event);
t.addChild(event);
t.setStatus(result.getException().getClass().getSimpleName());
} else {
t.setStatus(Message.SUCCESS);
}
return result;
} catch (RuntimeException e) {
Event event = null;
if (RpcException.class == e.getClass()) {
Throwable caseBy = e.getCause();
if (caseBy !=null && caseBy.getClass() == TimeoutException.class) {
event = Cat.newEvent("DUBBO_TIMEOUT_ERROR", loggerName);
} else {
event = Cat.newEvent("DUBBO_REMOTING_ERROR", loggerName);
}
} else {
event = Cat.newEvent("DUBBO_BIZ_ERROR", loggerName);
}
event.setStatus(e);
completeEvent(event);
t.addChild(event);
t.setStatus(e.getClass().getSimpleName());
if (result == null) {
throw e;
} else {
return result;
}
} finally {
t.complete();
CAT_CONTEXT.remove();
}
}
static class DubboCatContext implements Cat.Context {
private Map properties = new HashMap();
@Override
public void addProperty(String key, String value) {
properties.put(key,value);
}
@Override
public String getProperty(String key) {
return properties.get(key);
}
}
private void setAttachment(Cat.Context context) {
RpcContext.getContext().setAttachment(Cat.Context.ROOT,context.getProperty(Cat.Context.ROOT));
RpcContext.getContext().setAttachment(Cat.Context.CHILD,context.getProperty(Cat.Context.CHILD));
RpcContext.getContext().setAttachment(Cat.Context.PARENT,context.getProperty(Cat.Context.PARENT));
}
private Cat.Context getContext(){
Cat.Context context = CAT_CONTEXT.get();
if (context==null) {
context = initContext();
CAT_CONTEXT.set(context);
}
return context;
}
private Cat.Context initContext() {
Cat.Context context = new DubboCatContext();
Map attachments = RpcContext.getContext().getAttachments();
if (attachments!=null&&attachments.size()>0) {
for (Map.Entry entry:attachments.entrySet()) {
if (Cat.Context.CHILD.equals(entry.getKey()) || Cat.Context.ROOT.equals(entry.getKey()) || Cat.Context.PARENT.equals(entry.getKey())) {
context.addProperty(entry.getKey(),entry.getValue());
}
}
}
return context;
}
private void createConsumerCross(URL url, Transaction t) {
Event crossAppEvent = Cat.newEvent("PigeonCall.app", getProviderAppName(url));
Event crossServerEvent = Cat.newEvent("PigeonCall.server", url.getHost());
Event crossPortEvent = Cat.newEvent("PigeonCall.port", url.getPort() + "");
crossAppEvent.setStatus(Event.SUCCESS);
crossServerEvent.setStatus(Event.SUCCESS);
crossPortEvent.setStatus(Event.SUCCESS);
completeEvent(crossAppEvent);
completeEvent(crossPortEvent);
completeEvent(crossServerEvent);
t.addChild(crossAppEvent);
t.addChild(crossPortEvent);
t.addChild(crossServerEvent);
}
private void createProviderCross(URL url, Transaction t) {
String consumerAppName = RpcContext.getContext().getAttachment(Constants.APPLICATION_KEY);
if (StringUtils.isEmpty(consumerAppName)) {
consumerAppName = RpcContext.getContext().getRemoteHost() + ":" + RpcContext.getContext().getRemotePort();
}
Event crossAppEvent = Cat.newEvent("PigeonService.app", consumerAppName);
Event crossServerEvent = Cat.newEvent("PigeonService.client", url.getHost());
crossAppEvent.setStatus(Event.SUCCESS);
crossServerEvent.setStatus(Event.SUCCESS);
completeEvent(crossAppEvent);
completeEvent(crossServerEvent);
t.addChild(crossAppEvent);
t.addChild(crossServerEvent);
}
private void completeEvent(Event event) {
AbstractMessage message = (AbstractMessage) event;
message.setCompleted(true);
}
} 2）.以下是http方式部分代码实现：
CatHttpClientProxy.java
public void requestByGet(String url) {
Transaction t = Cat.newTransaction("PigeonCall", "method000");
//创建默认的httpClient实例
CloseableHttpClient httpClient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(5000).setConnectionRequestTimeout(1000)
.setSocketTimeout(5000).build();
try {
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(requestConfig);
//串联埋点
Cat.Context context = new CatHttpContext();
this.createConsumerCross(url, t);
Cat.logRemoteCallClient(context);
httpGet.setHeader(Cat.Context.ROOT, context.getProperty(Cat.Context.ROOT));
httpGet.setHeader(Cat.Context.PARENT, context.getProperty(Cat.Context.PARENT));
httpGet.setHeader(Cat.Context.CHILD, context.getProperty(Cat.Context.CHILD));
System.out.println("执行get请求:...." + httpGet.getURI());
CloseableHttpResponse httpResponse = null;
//发送get请求
httpResponse = httpClient.execute(httpGet);//请求返回的Resp，含http的header和执行结果实体Entity
try {
//response实体
HttpEntity entity = httpResponse.getEntity();//不包含header
if (null != entity) {
System.out.println("响应状态码:"+ httpResponse.getStatusLine());
System.out.println("-------------------------------------------------");
System.out.println("响应内容:" + EntityUtils.toString(entity));
}
} finally {
httpResponse.close();
}
t.setStatus(Transaction.SUCCESS);
} catch (Exception e) {
e.printStackTrace();
t.setStatus(e.getClass().getSimpleName());
} finally {
t.complete();
try {
closeHttpClient(httpClient);
} catch (IOException e) {
e.printStackTrace();
}
}
}
private void createConsumerCross(String url, Transaction t){
Event crossAppEvent = Cat.newEvent("PigeonCall.app", "serverName");
Event crossServerEvent = Cat.newEvent("PigeonCall.server", "serverIp");
Event crossPortEvent = Cat.newEvent("PigeonCall.port", "serverPort");
crossAppEvent.setStatus(Event.SUCCESS);
crossServerEvent.setStatus(Event.SUCCESS);
crossPortEvent.setStatus(Event.SUCCESS);
completeEvent(crossAppEvent);
completeEvent(crossPortEvent);
completeEvent(crossServerEvent);
t.addChild(crossAppEvent);
t.addChild(crossPortEvent);
t.addChild(crossServerEvent);
}
private void completeEvent(Event event){
AbstractMessage message = (AbstractMessage) event;
message.setCompleted(true);
}
private void closeHttpClient(CloseableHttpClient client) throws IOException{
if (client != null) {
client.close();
}
}------------------------------------------------------------------
2、CAT对redis缓存进行详细监控
CAT源码内部对于缓存的识别存在一个convention约定，是基于匹配“Cache.”字符串的，并且通过判断字符串“Cache.memcached”来支持memcached监控，可是没有对redis做显示支持，需要修改源码，增加判断字符串“Cache.redis”;
1).修改类：cat-home - com.dianping.cat.report.page.statistics.task.utilization.TransactionReportVisitor.java
增加对redis的判断支持：
private static final String REDIS = "Cache.redis";
public TransactionReportVisitor() {
m_types.add("URL");
m_types.add("Service");
m_types.add("PigeonService");
m_types.add("Call");
m_types.add("PigeonCall");
m_types.add("SQL");
m_types.add(MEMCACHED);
m_types.add(REDIS);
}
@Override
public void visitType(TransactionType type) {
String typeName = type.getId();
Domain domain = m_report.findOrCreateDomain(m_domain);
if ("Service".equals(typeName)) {
typeName = "PigeonService";
} else if ("Call".equals(typeName)) {
typeName = "PigeonCall";
} else if (typeName.startsWith(MEMCACHED)) {
typeName = MEMCACHED;
} else if (typeName.startsWith(REDIS)){
typeName = REDIS;
}
......
}
2).修改类：cat-core - com.dianping.cat.config.server.ServerConfigManager.java
增加对redis的判断支持：
public boolean isCacheTransaction(String type) {
return StringUtils.isNotEmpty(type) && (type.startsWith("Cache.memcached") || type.startsWith("Cache.redis"));
}
3).修改类：cat-consumer - com.dianping.cat.consumer.storage.StorageAnalyzer.java
增加对redis的判断支持：
private void processCacheTransaction(MessageTree tree, Transaction t) {
String cachePrefix = "Cache.";
String ip = "Default";
String domain = tree.getDomain();
String cacheType = t.getType().substring(cachePrefix.length());
String name = t.getName();
String method = name.substring(name.lastIndexOf(":") + 1);
List messages = t.getChildren();
for (Message message : messages) {
if (message instanceof Event) {
String type = message.getType();
if (type.equals("Cache.memcached.server") || type.equals("Cache.redis.server")) {
ip = message.getName();
int index = ip.indexOf(":");
if (index > -1) {
ip = ip.substring(0, index);
}
}
}
}
......
}
3、CAT对DB数据库进行详细监控
如果你的 orm 框架使用的mybatis，可以考虑通过实现拦截器 Interceptor 来对DB进行底层监控，CAT对数据库的埋点也存在convention，这里代码中存在hard code。具体埋点如下：
MappedStatement mappedStatement = (MappedStatement) invocation.getArgs()[0]; //得到类名，方法 String[] strArr = mappedStatement.getId().split("\\."); String methodName = strArr[strArr.length - 2] + "." + strArr[strArr.length - 1] Transaction t = Cat.newTransaction("SQL", "methodName"); //获取SQL类型 SqlCommandType sqlCommandType = mappedStatement.getSqlCommandType(); Cat.logEvent("SQL.Method", sqlCommandType.name().toLowerCase()); String JDBC_CONNECTION = "jdbc:mysql://unknown:3306/%s?useUnicode=true"; Cat.logEvent("SQL.Database", String.format(JDBC_CONNECTION, serverIp, dbName));
spring配置如下：
以上仅罗列了每个问题的一种实现方案，只是给正在研究CAT的同学一个参考思路，个人研究CAT也是刚开始，所说之处不免存在一些纰漏，欢迎指正和交流。