关于压测客户端
netty nio压测端
package com.nio.test;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequestEncoder;
import io.netty.handler.codec.http.HttpResponseDecoder;
import io.netty.handler.codec.http.HttpVersion;
import java.net.URI;
import java.nio.charset.StandardCharsets;
public class HttpClient {
public void connect(String host, int port) throws Exception {
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(workerGroup);
b.channel(NioSocketChannel.class);
b.option(ChannelOption.SO_KEEPALIVE, true);
b.handler(new ChannelInitializer() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
// 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
ch.pipeline().addLast(new HttpResponseDecoder());
// 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
ch.pipeline().addLast(new HttpRequestEncoder());
ch.pipeline().addLast(new HttpClientInboundHandler());
}
});
ChannelFuture f = b.connect(host, port).sync();
URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
String msg = "Are you ok?";
DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET,
uri.toASCIIString(), Unpooled.wrappedBuffer(msg.getBytes("UTF-8")));
// 构建http请求
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, request.content().readableBytes());
// 发送http请求
f.channel().write(request);
f.channel().flush();
f.channel().closeFuture().sync();
// }
} finally {
workerGroup.shutdownGracefully();
}
}
public void connect_post(String host, int port) throws Exception {
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(workerGroup);
b.channel(NioSocketChannel.class);
b.option(ChannelOption.SO_KEEPALIVE, true);
b.handler(new ChannelInitializer() {
@Override
public void initChannel(SocketChannel ch) throws Exception {
// 客户端接收到的是httpResponse响应，所以要使用HttpResponseDecoder进行解码
ch.pipeline().addLast(new HttpResponseDecoder());
// 客户端发送的是httprequest，所以要使用HttpRequestEncoder进行编码
ch.pipeline().addLast(new HttpRequestEncoder());
ch.pipeline().addLast(new HttpClientInboundHandler());
}
});
ChannelFuture f = b.connect(host, port).sync();
URI uri = new URI("http://gc.ditu.aliyun.com:80/geocoding?a=深圳市");
FullHttpRequest request = new DefaultFullHttpRequest(
HttpVersion.HTTP_1_1, HttpMethod.POST, uri.getRawPath());
// 构建http请求
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); // or HttpHeaders.Values.CLOSE
request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);
request.headers().add(HttpHeaders.Names.CONTENT_TYPE, "application/json");
ByteBuf bbuf = Unpooled.copiedBuffer("{\"jsonrpc\":\"2.0\",\"method\":\"calc.add\",\"params\":[1,2],\"id\":1}", StandardCharsets.UTF_8);
request.headers().set(HttpHeaders.Names.CONTENT_LENGTH, bbuf.readableBytes());
// 发送http请求
f.channel().write(request);
f.channel().flush();
f.channel().closeFuture().sync();
// }
} finally {
workerGroup.shutdownGracefully();
}
}
public static void main(String[] args) throws Exception {
HttpClient client = new HttpClient();
//请自行修改成服务端的IP
for(int k=0; k<1;k++){
client.connect("http://gc.ditu.aliyun.com/", 80);
System.out.println(k);
}
}
}
监听代码
package com.nio.test;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpResponse;
public class HttpClientInboundHandler extends ChannelInboundHandlerAdapter {
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
System.out.print("success!~!~");
if (msg instanceof HttpResponse)
{
HttpResponse response = (HttpResponse) msg;
// System.out.println("CONTENT_TYPE:" + response.headers().get(HttpHeaders.Names.CONTENT_TYPE));
System.out.println("HTTP_CODE:" + response.getStatus());
}
if(msg instanceof HttpContent)
{
HttpContent content = (HttpContent)msg;
ByteBuf buf = content.content();
System.out.println(buf.toString(io.netty.util.CharsetUtil.UTF_8));
buf.release();
}
ctx.close();
}
}
python tornado异步框架压测
import tornado.ioloop
from tornado.httpclient import AsyncHTTPClient
def handle_request(response):
if response.error:
print ("Error:", response.error)
else:
print(response.body)
param = {"msg":"111"}
param["img_msg"] = open("t.jpg",'r',encoding='utf-8')
url = 'http://gc.ditu.aliyun.com/geocoding?a=苏州市'
i = 1
print(param)
req = tornado.httpclient.HTTPRequest(url, 'POST', body=str(param))
http_client = AsyncHTTPClient()
while i<10000:
i += 1
http_client.fetch(req, handle_request)
tornado.ioloop.IOLoop.instance().start()
python 协程压测端
#-*- coding: utf-8 -*-
import urllib.request
# url = "http://r.qzone.qq.com/cgi-bin/user/cgi_personal_card?uin=284772894"
url = "http://gc.ditu.aliyun.com/geocoding?a="+urllib.request.quote("苏州市")
HEADER = {'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:29.0) Gecko/20100101 Firefox/29.0'}
# 自定义请求的方法,get post
def f(url):
import urllib
import urllib.request
data=urllib.request.urlopen(url).read()
z_data=data.decode('UTF-8')
print(z_data)
# 我自己封装了gevent 的方法,重载了run
from gevent import Greenlet
class MyGreenlet(Greenlet):
def __init__(self, func):
Greenlet.__init__(self)
self.func = func
def _run(self):
# gevent.sleep(self.n)
self.func
count = 3
green_let = []
for i in range(0, count):
green_let.append(MyGreenlet(f(url)))
for j in range(0, count):
green_let[j].start()
for k in range(0, count):
green_let[k].join()
** 当然也可以用多线程，apache ab等，如果是大量并发可以使用netty nio压测+虚拟机的方式（其实原理差不多，协程，tornado等）**
远程监控
- 把此shell脚本上传到服务器
#!/bin/bash
if (( $# != 1))
then
echo "please input sum times:"
exit 1
fi
sar -u 1 $1 | sed '1,3d'|sed '$d' > sar_cpu_1.txt
sar -r 1 $1 |sed '1,3d'|sed '$d' > sar_men_1.txt
sar -b 1 $1 |sed '1,3d'|sed '$d' > sar_io_1.txt
- 远程上传shell脚本，执行，下载日志
import paramiko
import paramiko
server_ip = '192.168.1.1'
server_user = 'root'
server_passwd = ''
server_port = 22
ssh = paramiko.SSHClient()
def ssh_connect():
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(server_ip, server_port,server_user, server_passwd)
return ssh
def client_connect():
client = paramiko.Transport((server_ip, server_port))
client.connect(username = server_user, password = server_passwd)
return client
def ssh_disconnect(client):
client.close()
def exec_cmd(command, ssh):
'''
windows客户端远程执行linux服务器上命令
'''
stdin, stdout, stderr = ssh.exec_command(command)
err = stderr.readline()
out = stdout.readline()
print(stdout.read())
def win_to_linux(localpath, remotepath,client):
'''
windows向linux服务器上传文件.
localpath 为本地文件的绝对路径。如：D: est.py
remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
'''
sftp = paramiko.SFTPClient.from_transport(client)
sftp.put(localpath,remotepath)
client.close()
def linux_to_win(localpath, remotepath,client):
'''
从linux服务器下载文件到本地
localpath 为本地文件的绝对路径。如：D: est.py
remotepath 为服务器端存放上传文件的绝对路径,而不是一个目录。如：/tmp/my_file.txt
'''
sftp = paramiko.SFTPClient.from_transport(client)
sftp.get(remotepath, localpath)
client.close()
class AllowAllKeys(paramiko.MissingHostKeyPolicy):
def missing_host_key(self, client, hostname, key):
return
def muit_exec_cmd(ssh,cmd):
'''
ssh ssh连接
cmd 多命名
'''
ssh.set_missing_host_key_policy(AllowAllKeys())
channel = ssh.invoke_shell()
stdin = channel.makefile('wb')
stdout = channel.makefile('rb')
stdin.write(cmd)
print(stdout.read())
stdout.close()
stdin.close()
cl = client_connect()
sh = ssh_connect()
win_to_linux("get_men_cpu.sh","/data/get_men_cpu.sh",cl)
exec_cmd("/data/get_men_cpu.sh 100",sh)
# 要和服务器那边的监控时间结束同步，可以设置下sleep时间
linux_to_win("d:/cpu.txt","/data/sar_cpu_1.txt",cl)
linux_to_win("d:/men.txt","/data/sar_men_1.txt",cl)
linux_to_win("d:/io.txt","/data/sar_io_1.txt",cl)
cl.close()
sh.close()
- js分析日志文件.cpu,内存,io都是基于这样分析
用的highcharts插件，自己去学习吧
Paste_Image.png
Paste_Image.png
Paste_Image.png
更新2015-7-15
- 监控各种中间件，数据库占用信息
top -u oracle>ora.txt
top -u mysql>ora.txt
更多参考
- tornado 压力测试
- ab压测
- java netty IO压测
- python gevent协程压测
- python多进程+协程压测端
- 多线程压测
- js分析服务器日志
- 监控服务器中间件